LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   if (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   if (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// isScalarToVector - Return true if the specified node is a
00200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00201 /// element is not an undef.
00202 bool ISD::isScalarToVector(const SDNode *N) {
00203   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00204     return true;
00205 
00206   if (N->getOpcode() != ISD::BUILD_VECTOR)
00207     return false;
00208   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00209     return false;
00210   unsigned NumElems = N->getNumOperands();
00211   if (NumElems == 1)
00212     return false;
00213   for (unsigned i = 1; i < NumElems; ++i) {
00214     SDValue V = N->getOperand(i);
00215     if (V.getOpcode() != ISD::UNDEF)
00216       return false;
00217   }
00218   return true;
00219 }
00220 
00221 /// allOperandsUndef - Return true if the node has at least one operand
00222 /// and all operands of the specified node are ISD::UNDEF.
00223 bool ISD::allOperandsUndef(const SDNode *N) {
00224   // Return false if the node has no operands.
00225   // This is "logically inconsistent" with the definition of "all" but
00226   // is probably the desired behavior.
00227   if (N->getNumOperands() == 0)
00228     return false;
00229 
00230   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00231     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00232       return false;
00233 
00234   return true;
00235 }
00236 
00237 ISD::NodeType ISD::getExtForLoadExtType(ISD::LoadExtType ExtType) {
00238   switch (ExtType) {
00239   case ISD::EXTLOAD:
00240     return ISD::ANY_EXTEND;
00241   case ISD::SEXTLOAD:
00242     return ISD::SIGN_EXTEND;
00243   case ISD::ZEXTLOAD:
00244     return ISD::ZERO_EXTEND;
00245   default:
00246     break;
00247   }
00248 
00249   llvm_unreachable("Invalid LoadExtType");
00250 }
00251 
00252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00253 /// when given the operation for (X op Y).
00254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00255   // To perform this operation, we just need to swap the L and G bits of the
00256   // operation.
00257   unsigned OldL = (Operation >> 2) & 1;
00258   unsigned OldG = (Operation >> 1) & 1;
00259   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00260                        (OldL << 1) |       // New G bit
00261                        (OldG << 2));       // New L bit.
00262 }
00263 
00264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00265 /// 'op' is a valid SetCC operation.
00266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00267   unsigned Operation = Op;
00268   if (isInteger)
00269     Operation ^= 7;   // Flip L, G, E bits, but not U.
00270   else
00271     Operation ^= 15;  // Flip all of the condition bits.
00272 
00273   if (Operation > ISD::SETTRUE2)
00274     Operation &= ~8;  // Don't let N and U bits get set.
00275 
00276   return ISD::CondCode(Operation);
00277 }
00278 
00279 
00280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00281 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00282 /// if the operation does not depend on the sign of the input (setne and seteq).
00283 static int isSignedOp(ISD::CondCode Opcode) {
00284   switch (Opcode) {
00285   default: llvm_unreachable("Illegal integer setcc operation!");
00286   case ISD::SETEQ:
00287   case ISD::SETNE: return 0;
00288   case ISD::SETLT:
00289   case ISD::SETLE:
00290   case ISD::SETGT:
00291   case ISD::SETGE: return 1;
00292   case ISD::SETULT:
00293   case ISD::SETULE:
00294   case ISD::SETUGT:
00295   case ISD::SETUGE: return 2;
00296   }
00297 }
00298 
00299 /// getSetCCOrOperation - Return the result of a logical OR between different
00300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00301 /// returns SETCC_INVALID if it is not possible to represent the resultant
00302 /// comparison.
00303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00304                                        bool isInteger) {
00305   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00306     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00307     return ISD::SETCC_INVALID;
00308 
00309   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00310 
00311   // If the N and U bits get set then the resultant comparison DOES suddenly
00312   // care about orderedness, and is true when ordered.
00313   if (Op > ISD::SETTRUE2)
00314     Op &= ~16;     // Clear the U bit if the N bit is set.
00315 
00316   // Canonicalize illegal integer setcc's.
00317   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00318     Op = ISD::SETNE;
00319 
00320   return ISD::CondCode(Op);
00321 }
00322 
00323 /// getSetCCAndOperation - Return the result of a logical AND between different
00324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00325 /// function returns zero if it is not possible to represent the resultant
00326 /// comparison.
00327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00328                                         bool isInteger) {
00329   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00330     // Cannot fold a signed setcc with an unsigned setcc.
00331     return ISD::SETCC_INVALID;
00332 
00333   // Combine all of the condition bits.
00334   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00335 
00336   // Canonicalize illegal integer setcc's.
00337   if (isInteger) {
00338     switch (Result) {
00339     default: break;
00340     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00341     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00342     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00343     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00344     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00345     }
00346   }
00347 
00348   return Result;
00349 }
00350 
00351 //===----------------------------------------------------------------------===//
00352 //                           SDNode Profile Support
00353 //===----------------------------------------------------------------------===//
00354 
00355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00356 ///
00357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00358   ID.AddInteger(OpC);
00359 }
00360 
00361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00362 /// solely with their pointer.
00363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00364   ID.AddPointer(VTList.VTs);
00365 }
00366 
00367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00368 ///
00369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00370                               ArrayRef<SDValue> Ops) {
00371   for (auto& Op : Ops) {
00372     ID.AddPointer(Op.getNode());
00373     ID.AddInteger(Op.getResNo());
00374   }
00375 }
00376 
00377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00378 ///
00379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00380                               ArrayRef<SDUse> Ops) {
00381   for (auto& Op : Ops) {
00382     ID.AddPointer(Op.getNode());
00383     ID.AddInteger(Op.getResNo());
00384   }
00385 }
00386 
00387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00388                                   bool exact) {
00389   ID.AddBoolean(nuw);
00390   ID.AddBoolean(nsw);
00391   ID.AddBoolean(exact);
00392 }
00393 
00394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00396                                   bool nuw, bool nsw, bool exact) {
00397   if (isBinOpWithFlags(Opcode))
00398     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00399 }
00400 
00401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00402                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00403   AddNodeIDOpcode(ID, OpC);
00404   AddNodeIDValueTypes(ID, VTList);
00405   AddNodeIDOperands(ID, OpList);
00406 }
00407 
00408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00409 /// the NodeID data.
00410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00411   switch (N->getOpcode()) {
00412   case ISD::TargetExternalSymbol:
00413   case ISD::ExternalSymbol:
00414     llvm_unreachable("Should only be used on nodes with operands");
00415   default: break;  // Normal nodes don't need extra info.
00416   case ISD::TargetConstant:
00417   case ISD::Constant: {
00418     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00419     ID.AddPointer(C->getConstantIntValue());
00420     ID.AddBoolean(C->isOpaque());
00421     break;
00422   }
00423   case ISD::TargetConstantFP:
00424   case ISD::ConstantFP: {
00425     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00426     break;
00427   }
00428   case ISD::TargetGlobalAddress:
00429   case ISD::GlobalAddress:
00430   case ISD::TargetGlobalTLSAddress:
00431   case ISD::GlobalTLSAddress: {
00432     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00433     ID.AddPointer(GA->getGlobal());
00434     ID.AddInteger(GA->getOffset());
00435     ID.AddInteger(GA->getTargetFlags());
00436     ID.AddInteger(GA->getAddressSpace());
00437     break;
00438   }
00439   case ISD::BasicBlock:
00440     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00441     break;
00442   case ISD::Register:
00443     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00444     break;
00445   case ISD::RegisterMask:
00446     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00447     break;
00448   case ISD::SRCVALUE:
00449     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00450     break;
00451   case ISD::FrameIndex:
00452   case ISD::TargetFrameIndex:
00453     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00454     break;
00455   case ISD::JumpTable:
00456   case ISD::TargetJumpTable:
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00458     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00459     break;
00460   case ISD::ConstantPool:
00461   case ISD::TargetConstantPool: {
00462     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00463     ID.AddInteger(CP->getAlignment());
00464     ID.AddInteger(CP->getOffset());
00465     if (CP->isMachineConstantPoolEntry())
00466       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00467     else
00468       ID.AddPointer(CP->getConstVal());
00469     ID.AddInteger(CP->getTargetFlags());
00470     break;
00471   }
00472   case ISD::TargetIndex: {
00473     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00474     ID.AddInteger(TI->getIndex());
00475     ID.AddInteger(TI->getOffset());
00476     ID.AddInteger(TI->getTargetFlags());
00477     break;
00478   }
00479   case ISD::LOAD: {
00480     const LoadSDNode *LD = cast<LoadSDNode>(N);
00481     ID.AddInteger(LD->getMemoryVT().getRawBits());
00482     ID.AddInteger(LD->getRawSubclassData());
00483     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00484     break;
00485   }
00486   case ISD::STORE: {
00487     const StoreSDNode *ST = cast<StoreSDNode>(N);
00488     ID.AddInteger(ST->getMemoryVT().getRawBits());
00489     ID.AddInteger(ST->getRawSubclassData());
00490     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00491     break;
00492   }
00493   case ISD::SDIV:
00494   case ISD::UDIV:
00495   case ISD::SRA:
00496   case ISD::SRL:
00497   case ISD::MUL:
00498   case ISD::ADD:
00499   case ISD::SUB:
00500   case ISD::SHL: {
00501     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00502     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00503                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00504     break;
00505   }
00506   case ISD::ATOMIC_CMP_SWAP:
00507   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00508   case ISD::ATOMIC_SWAP:
00509   case ISD::ATOMIC_LOAD_ADD:
00510   case ISD::ATOMIC_LOAD_SUB:
00511   case ISD::ATOMIC_LOAD_AND:
00512   case ISD::ATOMIC_LOAD_OR:
00513   case ISD::ATOMIC_LOAD_XOR:
00514   case ISD::ATOMIC_LOAD_NAND:
00515   case ISD::ATOMIC_LOAD_MIN:
00516   case ISD::ATOMIC_LOAD_MAX:
00517   case ISD::ATOMIC_LOAD_UMIN:
00518   case ISD::ATOMIC_LOAD_UMAX:
00519   case ISD::ATOMIC_LOAD:
00520   case ISD::ATOMIC_STORE: {
00521     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00522     ID.AddInteger(AT->getMemoryVT().getRawBits());
00523     ID.AddInteger(AT->getRawSubclassData());
00524     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00525     break;
00526   }
00527   case ISD::PREFETCH: {
00528     const MemSDNode *PF = cast<MemSDNode>(N);
00529     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00530     break;
00531   }
00532   case ISD::VECTOR_SHUFFLE: {
00533     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00534     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00535          i != e; ++i)
00536       ID.AddInteger(SVN->getMaskElt(i));
00537     break;
00538   }
00539   case ISD::TargetBlockAddress:
00540   case ISD::BlockAddress: {
00541     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00542     ID.AddPointer(BA->getBlockAddress());
00543     ID.AddInteger(BA->getOffset());
00544     ID.AddInteger(BA->getTargetFlags());
00545     break;
00546   }
00547   } // end switch (N->getOpcode())
00548 
00549   // Target specific memory nodes could also have address spaces to check.
00550   if (N->isTargetMemoryOpcode())
00551     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00552 }
00553 
00554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00555 /// data.
00556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00557   AddNodeIDOpcode(ID, N->getOpcode());
00558   // Add the return value info.
00559   AddNodeIDValueTypes(ID, N->getVTList());
00560   // Add the operand info.
00561   AddNodeIDOperands(ID, N->ops());
00562 
00563   // Handle SDNode leafs with special info.
00564   AddNodeIDCustom(ID, N);
00565 }
00566 
00567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00568 /// the CSE map that carries volatility, temporalness, indexing mode, and
00569 /// extension/truncation information.
00570 ///
00571 static inline unsigned
00572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00573                      bool isNonTemporal, bool isInvariant) {
00574   assert((ConvType & 3) == ConvType &&
00575          "ConvType may not require more than 2 bits!");
00576   assert((AM & 7) == AM &&
00577          "AM may not require more than 3 bits!");
00578   return ConvType |
00579          (AM << 2) |
00580          (isVolatile << 5) |
00581          (isNonTemporal << 6) |
00582          (isInvariant << 7);
00583 }
00584 
00585 //===----------------------------------------------------------------------===//
00586 //                              SelectionDAG Class
00587 //===----------------------------------------------------------------------===//
00588 
00589 /// doNotCSE - Return true if CSE should not be performed for this node.
00590 static bool doNotCSE(SDNode *N) {
00591   if (N->getValueType(0) == MVT::Glue)
00592     return true; // Never CSE anything that produces a flag.
00593 
00594   switch (N->getOpcode()) {
00595   default: break;
00596   case ISD::HANDLENODE:
00597   case ISD::EH_LABEL:
00598     return true;   // Never CSE these nodes.
00599   }
00600 
00601   // Check that remaining values produced are not flags.
00602   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00603     if (N->getValueType(i) == MVT::Glue)
00604       return true; // Never CSE anything that produces a flag.
00605 
00606   return false;
00607 }
00608 
00609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00610 /// SelectionDAG.
00611 void SelectionDAG::RemoveDeadNodes() {
00612   // Create a dummy node (which is not added to allnodes), that adds a reference
00613   // to the root node, preventing it from being deleted.
00614   HandleSDNode Dummy(getRoot());
00615 
00616   SmallVector<SDNode*, 128> DeadNodes;
00617 
00618   // Add all obviously-dead nodes to the DeadNodes worklist.
00619   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00620     if (I->use_empty())
00621       DeadNodes.push_back(I);
00622 
00623   RemoveDeadNodes(DeadNodes);
00624 
00625   // If the root changed (e.g. it was a dead load, update the root).
00626   setRoot(Dummy.getValue());
00627 }
00628 
00629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00630 /// given list, and any nodes that become unreachable as a result.
00631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00632 
00633   // Process the worklist, deleting the nodes and adding their uses to the
00634   // worklist.
00635   while (!DeadNodes.empty()) {
00636     SDNode *N = DeadNodes.pop_back_val();
00637 
00638     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00639       DUL->NodeDeleted(N, nullptr);
00640 
00641     // Take the node out of the appropriate CSE map.
00642     RemoveNodeFromCSEMaps(N);
00643 
00644     // Next, brutally remove the operand list.  This is safe to do, as there are
00645     // no cycles in the graph.
00646     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00647       SDUse &Use = *I++;
00648       SDNode *Operand = Use.getNode();
00649       Use.set(SDValue());
00650 
00651       // Now that we removed this operand, see if there are no uses of it left.
00652       if (Operand->use_empty())
00653         DeadNodes.push_back(Operand);
00654     }
00655 
00656     DeallocateNode(N);
00657   }
00658 }
00659 
00660 void SelectionDAG::RemoveDeadNode(SDNode *N){
00661   SmallVector<SDNode*, 16> DeadNodes(1, N);
00662 
00663   // Create a dummy node that adds a reference to the root node, preventing
00664   // it from being deleted.  (This matters if the root is an operand of the
00665   // dead node.)
00666   HandleSDNode Dummy(getRoot());
00667 
00668   RemoveDeadNodes(DeadNodes);
00669 }
00670 
00671 void SelectionDAG::DeleteNode(SDNode *N) {
00672   // First take this out of the appropriate CSE map.
00673   RemoveNodeFromCSEMaps(N);
00674 
00675   // Finally, remove uses due to operands of this node, remove from the
00676   // AllNodes list, and delete the node.
00677   DeleteNodeNotInCSEMaps(N);
00678 }
00679 
00680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00681   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00682   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00683 
00684   // Drop all of the operands and decrement used node's use counts.
00685   N->DropOperands();
00686 
00687   DeallocateNode(N);
00688 }
00689 
00690 void SelectionDAG::DeallocateNode(SDNode *N) {
00691   if (N->OperandsNeedDelete)
00692     delete[] N->OperandList;
00693 
00694   // Set the opcode to DELETED_NODE to help catch bugs when node
00695   // memory is reallocated.
00696   N->NodeType = ISD::DELETED_NODE;
00697 
00698   NodeAllocator.Deallocate(AllNodes.remove(N));
00699 
00700   // If any of the SDDbgValue nodes refer to this SDNode, invalidate them.
00701   ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N);
00702   for (unsigned i = 0, e = DbgVals.size(); i != e; ++i)
00703     DbgVals[i]->setIsInvalidated();
00704 }
00705 
00706 #ifndef NDEBUG
00707 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00708 static void VerifySDNode(SDNode *N) {
00709   switch (N->getOpcode()) {
00710   default:
00711     break;
00712   case ISD::BUILD_PAIR: {
00713     EVT VT = N->getValueType(0);
00714     assert(N->getNumValues() == 1 && "Too many results!");
00715     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00716            "Wrong return type!");
00717     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00718     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00719            "Mismatched operand types!");
00720     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00721            "Wrong operand type!");
00722     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00723            "Wrong return type size");
00724     break;
00725   }
00726   case ISD::BUILD_VECTOR: {
00727     assert(N->getNumValues() == 1 && "Too many results!");
00728     assert(N->getValueType(0).isVector() && "Wrong return type!");
00729     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00730            "Wrong number of operands!");
00731     EVT EltVT = N->getValueType(0).getVectorElementType();
00732     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00733       assert((I->getValueType() == EltVT ||
00734              (EltVT.isInteger() && I->getValueType().isInteger() &&
00735               EltVT.bitsLE(I->getValueType()))) &&
00736             "Wrong operand type!");
00737       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00738              "Operands must all have the same type");
00739     }
00740     break;
00741   }
00742   }
00743 }
00744 #endif // NDEBUG
00745 
00746 /// \brief Insert a newly allocated node into the DAG.
00747 ///
00748 /// Handles insertion into the all nodes list and CSE map, as well as
00749 /// verification and other common operations when a new node is allocated.
00750 void SelectionDAG::InsertNode(SDNode *N) {
00751   AllNodes.push_back(N);
00752 #ifndef NDEBUG
00753   VerifySDNode(N);
00754 #endif
00755 }
00756 
00757 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00758 /// correspond to it.  This is useful when we're about to delete or repurpose
00759 /// the node.  We don't want future request for structurally identical nodes
00760 /// to return N anymore.
00761 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00762   bool Erased = false;
00763   switch (N->getOpcode()) {
00764   case ISD::HANDLENODE: return false;  // noop.
00765   case ISD::CONDCODE:
00766     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00767            "Cond code doesn't exist!");
00768     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00769     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00770     break;
00771   case ISD::ExternalSymbol:
00772     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00773     break;
00774   case ISD::TargetExternalSymbol: {
00775     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00776     Erased = TargetExternalSymbols.erase(
00777                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00778                                                     ESN->getTargetFlags()));
00779     break;
00780   }
00781   case ISD::VALUETYPE: {
00782     EVT VT = cast<VTSDNode>(N)->getVT();
00783     if (VT.isExtended()) {
00784       Erased = ExtendedValueTypeNodes.erase(VT);
00785     } else {
00786       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00787       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00788     }
00789     break;
00790   }
00791   default:
00792     // Remove it from the CSE Map.
00793     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00794     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00795     Erased = CSEMap.RemoveNode(N);
00796     break;
00797   }
00798 #ifndef NDEBUG
00799   // Verify that the node was actually in one of the CSE maps, unless it has a
00800   // flag result (which cannot be CSE'd) or is one of the special cases that are
00801   // not subject to CSE.
00802   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00803       !N->isMachineOpcode() && !doNotCSE(N)) {
00804     N->dump(this);
00805     dbgs() << "\n";
00806     llvm_unreachable("Node is not in map!");
00807   }
00808 #endif
00809   return Erased;
00810 }
00811 
00812 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00813 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00814 /// node already exists, in which case transfer all its users to the existing
00815 /// node. This transfer can potentially trigger recursive merging.
00816 ///
00817 void
00818 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00819   // For node types that aren't CSE'd, just act as if no identical node
00820   // already exists.
00821   if (!doNotCSE(N)) {
00822     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00823     if (Existing != N) {
00824       // If there was already an existing matching node, use ReplaceAllUsesWith
00825       // to replace the dead one with the existing one.  This can cause
00826       // recursive merging of other unrelated nodes down the line.
00827       ReplaceAllUsesWith(N, Existing);
00828 
00829       // N is now dead. Inform the listeners and delete it.
00830       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00831         DUL->NodeDeleted(N, Existing);
00832       DeleteNodeNotInCSEMaps(N);
00833       return;
00834     }
00835   }
00836 
00837   // If the node doesn't already exist, we updated it.  Inform listeners.
00838   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00839     DUL->NodeUpdated(N);
00840 }
00841 
00842 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00843 /// were replaced with those specified.  If this node is never memoized,
00844 /// return null, otherwise return a pointer to the slot it would take.  If a
00845 /// node already exists with these operands, the slot will be non-null.
00846 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00847                                            void *&InsertPos) {
00848   if (doNotCSE(N))
00849     return nullptr;
00850 
00851   SDValue Ops[] = { Op };
00852   FoldingSetNodeID ID;
00853   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00854   AddNodeIDCustom(ID, N);
00855   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00856   return Node;
00857 }
00858 
00859 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00860 /// were replaced with those specified.  If this node is never memoized,
00861 /// return null, otherwise return a pointer to the slot it would take.  If a
00862 /// node already exists with these operands, the slot will be non-null.
00863 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00864                                            SDValue Op1, SDValue Op2,
00865                                            void *&InsertPos) {
00866   if (doNotCSE(N))
00867     return nullptr;
00868 
00869   SDValue Ops[] = { Op1, Op2 };
00870   FoldingSetNodeID ID;
00871   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00872   AddNodeIDCustom(ID, N);
00873   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00874   return Node;
00875 }
00876 
00877 
00878 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00879 /// were replaced with those specified.  If this node is never memoized,
00880 /// return null, otherwise return a pointer to the slot it would take.  If a
00881 /// node already exists with these operands, the slot will be non-null.
00882 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00883                                            void *&InsertPos) {
00884   if (doNotCSE(N))
00885     return nullptr;
00886 
00887   FoldingSetNodeID ID;
00888   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00889   AddNodeIDCustom(ID, N);
00890   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00891   return Node;
00892 }
00893 
00894 /// getEVTAlignment - Compute the default alignment value for the
00895 /// given type.
00896 ///
00897 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00898   Type *Ty = VT == MVT::iPTR ?
00899                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00900                    VT.getTypeForEVT(*getContext());
00901 
00902   return TM.getSubtargetImpl()
00903       ->getTargetLowering()
00904       ->getDataLayout()
00905       ->getABITypeAlignment(Ty);
00906 }
00907 
00908 // EntryNode could meaningfully have debug info if we can find it...
00909 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00910     : TM(tm), TSI(tm.getSubtargetImpl()->getSelectionDAGInfo()), TLI(nullptr),
00911       OptLevel(OL),
00912       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00913       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00914       UpdateListeners(nullptr) {
00915   AllNodes.push_back(&EntryNode);
00916   DbgInfo = new SDDbgInfo();
00917 }
00918 
00919 void SelectionDAG::init(MachineFunction &mf, const TargetLowering *tli) {
00920   MF = &mf;
00921   TLI = tli;
00922   Context = &mf.getFunction()->getContext();
00923 }
00924 
00925 SelectionDAG::~SelectionDAG() {
00926   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00927   allnodes_clear();
00928   delete DbgInfo;
00929 }
00930 
00931 void SelectionDAG::allnodes_clear() {
00932   assert(&*AllNodes.begin() == &EntryNode);
00933   AllNodes.remove(AllNodes.begin());
00934   while (!AllNodes.empty())
00935     DeallocateNode(AllNodes.begin());
00936 }
00937 
00938 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00939                                             SDVTList VTs, SDValue N1,
00940                                             SDValue N2, bool nuw, bool nsw,
00941                                             bool exact) {
00942   if (isBinOpWithFlags(Opcode)) {
00943     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00944         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00945     FN->setHasNoUnsignedWrap(nuw);
00946     FN->setHasNoSignedWrap(nsw);
00947     FN->setIsExact(exact);
00948 
00949     return FN;
00950   }
00951 
00952   BinarySDNode *N = new (NodeAllocator)
00953       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00954   return N;
00955 }
00956 
00957 void SelectionDAG::clear() {
00958   allnodes_clear();
00959   OperandAllocator.Reset();
00960   CSEMap.clear();
00961 
00962   ExtendedValueTypeNodes.clear();
00963   ExternalSymbols.clear();
00964   TargetExternalSymbols.clear();
00965   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00966             static_cast<CondCodeSDNode*>(nullptr));
00967   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00968             static_cast<SDNode*>(nullptr));
00969 
00970   EntryNode.UseList = nullptr;
00971   AllNodes.push_back(&EntryNode);
00972   Root = getEntryNode();
00973   DbgInfo->clear();
00974 }
00975 
00976 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00977   return VT.bitsGT(Op.getValueType()) ?
00978     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00979     getNode(ISD::TRUNCATE, DL, VT, Op);
00980 }
00981 
00982 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00983   return VT.bitsGT(Op.getValueType()) ?
00984     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00985     getNode(ISD::TRUNCATE, DL, VT, Op);
00986 }
00987 
00988 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00989   return VT.bitsGT(Op.getValueType()) ?
00990     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00991     getNode(ISD::TRUNCATE, DL, VT, Op);
00992 }
00993 
00994 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
00995                                         EVT OpVT) {
00996   if (VT.bitsLE(Op.getValueType()))
00997     return getNode(ISD::TRUNCATE, SL, VT, Op);
00998 
00999   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
01000   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
01001 }
01002 
01003 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01004   assert(!VT.isVector() &&
01005          "getZeroExtendInReg should use the vector element type instead of "
01006          "the vector type!");
01007   if (Op.getValueType() == VT) return Op;
01008   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01009   APInt Imm = APInt::getLowBitsSet(BitWidth,
01010                                    VT.getSizeInBits());
01011   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01012                  getConstant(Imm, Op.getValueType()));
01013 }
01014 
01015 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01016   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01017   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01018          "The sizes of the input and result must match in order to perform the "
01019          "extend in-register.");
01020   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01021          "The destination vector type must have fewer lanes than the input.");
01022   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01023 }
01024 
01025 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01026   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01027   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01028          "The sizes of the input and result must match in order to perform the "
01029          "extend in-register.");
01030   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01031          "The destination vector type must have fewer lanes than the input.");
01032   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01033 }
01034 
01035 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01036   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01037   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01038          "The sizes of the input and result must match in order to perform the "
01039          "extend in-register.");
01040   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01041          "The destination vector type must have fewer lanes than the input.");
01042   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01043 }
01044 
01045 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01046 ///
01047 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01048   EVT EltVT = VT.getScalarType();
01049   SDValue NegOne =
01050     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01051   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01052 }
01053 
01054 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01055   EVT EltVT = VT.getScalarType();
01056   SDValue TrueValue;
01057   switch (TLI->getBooleanContents(VT)) {
01058     case TargetLowering::ZeroOrOneBooleanContent:
01059     case TargetLowering::UndefinedBooleanContent:
01060       TrueValue = getConstant(1, VT);
01061       break;
01062     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01063       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01064                               VT);
01065       break;
01066   }
01067   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01068 }
01069 
01070 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01071   EVT EltVT = VT.getScalarType();
01072   assert((EltVT.getSizeInBits() >= 64 ||
01073          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01074          "getConstant with a uint64_t value that doesn't fit in the type!");
01075   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01076 }
01077 
01078 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01079 {
01080   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01081 }
01082 
01083 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01084                                   bool isO) {
01085   assert(VT.isInteger() && "Cannot create FP integer constant!");
01086 
01087   EVT EltVT = VT.getScalarType();
01088   const ConstantInt *Elt = &Val;
01089 
01090   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01091 
01092   // In some cases the vector type is legal but the element type is illegal and
01093   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01094   // inserted value (the type does not need to match the vector element type).
01095   // Any extra bits introduced will be truncated away.
01096   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01097       TargetLowering::TypePromoteInteger) {
01098    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01099    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01100    Elt = ConstantInt::get(*getContext(), NewVal);
01101   }
01102   // In other cases the element type is illegal and needs to be expanded, for
01103   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01104   // the value into n parts and use a vector type with n-times the elements.
01105   // Then bitcast to the type requested.
01106   // Legalizing constants too early makes the DAGCombiner's job harder so we
01107   // only legalize if the DAG tells us we must produce legal types.
01108   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01109            TLI->getTypeAction(*getContext(), EltVT) ==
01110            TargetLowering::TypeExpandInteger) {
01111     APInt NewVal = Elt->getValue();
01112     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01113     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01114     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01115     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01116 
01117     // Check the temporary vector is the correct size. If this fails then
01118     // getTypeToTransformTo() probably returned a type whose size (in bits)
01119     // isn't a power-of-2 factor of the requested type size.
01120     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01121 
01122     SmallVector<SDValue, 2> EltParts;
01123     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01124       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01125                                            .trunc(ViaEltSizeInBits),
01126                                      ViaEltVT, isT, isO));
01127     }
01128 
01129     // EltParts is currently in little endian order. If we actually want
01130     // big-endian order then reverse it now.
01131     if (TLI->isBigEndian())
01132       std::reverse(EltParts.begin(), EltParts.end());
01133 
01134     // The elements must be reversed when the element order is different
01135     // to the endianness of the elements (because the BITCAST is itself a
01136     // vector shuffle in this situation). However, we do not need any code to
01137     // perform this reversal because getConstant() is producing a vector
01138     // splat.
01139     // This situation occurs in MIPS MSA.
01140 
01141     SmallVector<SDValue, 8> Ops;
01142     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01143       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01144 
01145     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01146                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01147                                      Ops));
01148     return Result;
01149   }
01150 
01151   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01152          "APInt size does not match type size!");
01153   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01154   FoldingSetNodeID ID;
01155   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01156   ID.AddPointer(Elt);
01157   ID.AddBoolean(isO);
01158   void *IP = nullptr;
01159   SDNode *N = nullptr;
01160   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01161     if (!VT.isVector())
01162       return SDValue(N, 0);
01163 
01164   if (!N) {
01165     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01166     CSEMap.InsertNode(N, IP);
01167     InsertNode(N);
01168   }
01169 
01170   SDValue Result(N, 0);
01171   if (VT.isVector()) {
01172     SmallVector<SDValue, 8> Ops;
01173     Ops.assign(VT.getVectorNumElements(), Result);
01174     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01175   }
01176   return Result;
01177 }
01178 
01179 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01180   return getConstant(Val,
01181                      TM.getSubtargetImpl()->getTargetLowering()->getPointerTy(),
01182                      isTarget);
01183 }
01184 
01185 
01186 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01187   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01188 }
01189 
01190 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01191   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01192 
01193   EVT EltVT = VT.getScalarType();
01194 
01195   // Do the map lookup using the actual bit pattern for the floating point
01196   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01197   // we don't have issues with SNANs.
01198   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01199   FoldingSetNodeID ID;
01200   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01201   ID.AddPointer(&V);
01202   void *IP = nullptr;
01203   SDNode *N = nullptr;
01204   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01205     if (!VT.isVector())
01206       return SDValue(N, 0);
01207 
01208   if (!N) {
01209     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01210     CSEMap.InsertNode(N, IP);
01211     InsertNode(N);
01212   }
01213 
01214   SDValue Result(N, 0);
01215   if (VT.isVector()) {
01216     SmallVector<SDValue, 8> Ops;
01217     Ops.assign(VT.getVectorNumElements(), Result);
01218     // FIXME SDLoc info might be appropriate here
01219     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01220   }
01221   return Result;
01222 }
01223 
01224 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01225   EVT EltVT = VT.getScalarType();
01226   if (EltVT==MVT::f32)
01227     return getConstantFP(APFloat((float)Val), VT, isTarget);
01228   else if (EltVT==MVT::f64)
01229     return getConstantFP(APFloat(Val), VT, isTarget);
01230   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01231            EltVT==MVT::f16) {
01232     bool ignored;
01233     APFloat apf = APFloat(Val);
01234     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01235                 &ignored);
01236     return getConstantFP(apf, VT, isTarget);
01237   } else
01238     llvm_unreachable("Unsupported type in getConstantFP");
01239 }
01240 
01241 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01242                                        EVT VT, int64_t Offset,
01243                                        bool isTargetGA,
01244                                        unsigned char TargetFlags) {
01245   assert((TargetFlags == 0 || isTargetGA) &&
01246          "Cannot set target flags on target-independent globals");
01247   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01248 
01249   // Truncate (with sign-extension) the offset value to the pointer size.
01250   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01251   if (BitWidth < 64)
01252     Offset = SignExtend64(Offset, BitWidth);
01253 
01254   unsigned Opc;
01255   if (GV->isThreadLocal())
01256     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01257   else
01258     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01259 
01260   FoldingSetNodeID ID;
01261   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01262   ID.AddPointer(GV);
01263   ID.AddInteger(Offset);
01264   ID.AddInteger(TargetFlags);
01265   ID.AddInteger(GV->getType()->getAddressSpace());
01266   void *IP = nullptr;
01267   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01268     return SDValue(E, 0);
01269 
01270   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01271                                                       DL.getDebugLoc(), GV, VT,
01272                                                       Offset, TargetFlags);
01273   CSEMap.InsertNode(N, IP);
01274     InsertNode(N);
01275   return SDValue(N, 0);
01276 }
01277 
01278 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01279   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01280   FoldingSetNodeID ID;
01281   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01282   ID.AddInteger(FI);
01283   void *IP = nullptr;
01284   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01285     return SDValue(E, 0);
01286 
01287   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01288   CSEMap.InsertNode(N, IP);
01289   InsertNode(N);
01290   return SDValue(N, 0);
01291 }
01292 
01293 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01294                                    unsigned char TargetFlags) {
01295   assert((TargetFlags == 0 || isTarget) &&
01296          "Cannot set target flags on target-independent jump tables");
01297   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01298   FoldingSetNodeID ID;
01299   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01300   ID.AddInteger(JTI);
01301   ID.AddInteger(TargetFlags);
01302   void *IP = nullptr;
01303   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01304     return SDValue(E, 0);
01305 
01306   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01307                                                   TargetFlags);
01308   CSEMap.InsertNode(N, IP);
01309   InsertNode(N);
01310   return SDValue(N, 0);
01311 }
01312 
01313 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01314                                       unsigned Alignment, int Offset,
01315                                       bool isTarget,
01316                                       unsigned char TargetFlags) {
01317   assert((TargetFlags == 0 || isTarget) &&
01318          "Cannot set target flags on target-independent globals");
01319   if (Alignment == 0)
01320     Alignment = TM.getSubtargetImpl()
01321                     ->getTargetLowering()
01322                     ->getDataLayout()
01323                     ->getPrefTypeAlignment(C->getType());
01324   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01325   FoldingSetNodeID ID;
01326   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01327   ID.AddInteger(Alignment);
01328   ID.AddInteger(Offset);
01329   ID.AddPointer(C);
01330   ID.AddInteger(TargetFlags);
01331   void *IP = nullptr;
01332   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01333     return SDValue(E, 0);
01334 
01335   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01336                                                      Alignment, TargetFlags);
01337   CSEMap.InsertNode(N, IP);
01338   InsertNode(N);
01339   return SDValue(N, 0);
01340 }
01341 
01342 
01343 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01344                                       unsigned Alignment, int Offset,
01345                                       bool isTarget,
01346                                       unsigned char TargetFlags) {
01347   assert((TargetFlags == 0 || isTarget) &&
01348          "Cannot set target flags on target-independent globals");
01349   if (Alignment == 0)
01350     Alignment = TM.getSubtargetImpl()
01351                     ->getTargetLowering()
01352                     ->getDataLayout()
01353                     ->getPrefTypeAlignment(C->getType());
01354   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01355   FoldingSetNodeID ID;
01356   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01357   ID.AddInteger(Alignment);
01358   ID.AddInteger(Offset);
01359   C->addSelectionDAGCSEId(ID);
01360   ID.AddInteger(TargetFlags);
01361   void *IP = nullptr;
01362   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01363     return SDValue(E, 0);
01364 
01365   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01366                                                      Alignment, TargetFlags);
01367   CSEMap.InsertNode(N, IP);
01368   InsertNode(N);
01369   return SDValue(N, 0);
01370 }
01371 
01372 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01373                                      unsigned char TargetFlags) {
01374   FoldingSetNodeID ID;
01375   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01376   ID.AddInteger(Index);
01377   ID.AddInteger(Offset);
01378   ID.AddInteger(TargetFlags);
01379   void *IP = nullptr;
01380   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01381     return SDValue(E, 0);
01382 
01383   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01384                                                     TargetFlags);
01385   CSEMap.InsertNode(N, IP);
01386   InsertNode(N);
01387   return SDValue(N, 0);
01388 }
01389 
01390 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01391   FoldingSetNodeID ID;
01392   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01393   ID.AddPointer(MBB);
01394   void *IP = nullptr;
01395   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01396     return SDValue(E, 0);
01397 
01398   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01399   CSEMap.InsertNode(N, IP);
01400   InsertNode(N);
01401   return SDValue(N, 0);
01402 }
01403 
01404 SDValue SelectionDAG::getValueType(EVT VT) {
01405   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01406       ValueTypeNodes.size())
01407     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01408 
01409   SDNode *&N = VT.isExtended() ?
01410     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01411 
01412   if (N) return SDValue(N, 0);
01413   N = new (NodeAllocator) VTSDNode(VT);
01414   InsertNode(N);
01415   return SDValue(N, 0);
01416 }
01417 
01418 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01419   SDNode *&N = ExternalSymbols[Sym];
01420   if (N) return SDValue(N, 0);
01421   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01422   InsertNode(N);
01423   return SDValue(N, 0);
01424 }
01425 
01426 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01427                                               unsigned char TargetFlags) {
01428   SDNode *&N =
01429     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01430                                                                TargetFlags)];
01431   if (N) return SDValue(N, 0);
01432   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01433   InsertNode(N);
01434   return SDValue(N, 0);
01435 }
01436 
01437 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01438   if ((unsigned)Cond >= CondCodeNodes.size())
01439     CondCodeNodes.resize(Cond+1);
01440 
01441   if (!CondCodeNodes[Cond]) {
01442     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01443     CondCodeNodes[Cond] = N;
01444     InsertNode(N);
01445   }
01446 
01447   return SDValue(CondCodeNodes[Cond], 0);
01448 }
01449 
01450 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01451 // the shuffle mask M that point at N1 to point at N2, and indices that point
01452 // N2 to point at N1.
01453 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01454   std::swap(N1, N2);
01455   int NElts = M.size();
01456   for (int i = 0; i != NElts; ++i) {
01457     if (M[i] >= NElts)
01458       M[i] -= NElts;
01459     else if (M[i] >= 0)
01460       M[i] += NElts;
01461   }
01462 }
01463 
01464 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01465                                        SDValue N2, const int *Mask) {
01466   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01467          "Invalid VECTOR_SHUFFLE");
01468 
01469   // Canonicalize shuffle undef, undef -> undef
01470   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01471     return getUNDEF(VT);
01472 
01473   // Validate that all indices in Mask are within the range of the elements
01474   // input to the shuffle.
01475   unsigned NElts = VT.getVectorNumElements();
01476   SmallVector<int, 8> MaskVec;
01477   for (unsigned i = 0; i != NElts; ++i) {
01478     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01479     MaskVec.push_back(Mask[i]);
01480   }
01481 
01482   // Canonicalize shuffle v, v -> v, undef
01483   if (N1 == N2) {
01484     N2 = getUNDEF(VT);
01485     for (unsigned i = 0; i != NElts; ++i)
01486       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01487   }
01488 
01489   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01490   if (N1.getOpcode() == ISD::UNDEF)
01491     commuteShuffle(N1, N2, MaskVec);
01492 
01493   // Canonicalize all index into lhs, -> shuffle lhs, undef
01494   // Canonicalize all index into rhs, -> shuffle rhs, undef
01495   bool AllLHS = true, AllRHS = true;
01496   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01497   for (unsigned i = 0; i != NElts; ++i) {
01498     if (MaskVec[i] >= (int)NElts) {
01499       if (N2Undef)
01500         MaskVec[i] = -1;
01501       else
01502         AllLHS = false;
01503     } else if (MaskVec[i] >= 0) {
01504       AllRHS = false;
01505     }
01506   }
01507   if (AllLHS && AllRHS)
01508     return getUNDEF(VT);
01509   if (AllLHS && !N2Undef)
01510     N2 = getUNDEF(VT);
01511   if (AllRHS) {
01512     N1 = getUNDEF(VT);
01513     commuteShuffle(N1, N2, MaskVec);
01514   }
01515   // Reset our undef status after accounting for the mask.
01516   N2Undef = N2.getOpcode() == ISD::UNDEF;
01517   // Re-check whether both sides ended up undef.
01518   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01519     return getUNDEF(VT);
01520 
01521   // If Identity shuffle return that node.
01522   bool Identity = true;
01523   for (unsigned i = 0; i != NElts; ++i) {
01524     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01525   }
01526   if (Identity && NElts)
01527     return N1;
01528 
01529   // Shuffling a constant splat doesn't change the result.
01530   if (N2Undef) {
01531     SDValue V = N1;
01532 
01533     // Look through any bitcasts. We check that these don't change the number
01534     // (and size) of elements and just changes their types.
01535     while (V.getOpcode() == ISD::BITCAST)
01536       V = V->getOperand(0);
01537 
01538     // A splat should always show up as a build vector node.
01539     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01540       BitVector UndefElements;
01541       SDValue Splat = BV->getSplatValue(&UndefElements);
01542       // If this is a splat of an undef, shuffling it is also undef.
01543       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01544         return getUNDEF(VT);
01545 
01546       // We only have a splat which can skip shuffles if there is a splatted
01547       // value and no undef lanes rearranged by the shuffle.
01548       if (Splat && UndefElements.none()) {
01549         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01550         // number of elements match or the value splatted is a zero constant.
01551         if (V.getValueType().getVectorNumElements() ==
01552             VT.getVectorNumElements())
01553           return N1;
01554         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01555           if (C->isNullValue())
01556             return N1;
01557       }
01558     }
01559   }
01560 
01561   FoldingSetNodeID ID;
01562   SDValue Ops[2] = { N1, N2 };
01563   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01564   for (unsigned i = 0; i != NElts; ++i)
01565     ID.AddInteger(MaskVec[i]);
01566 
01567   void* IP = nullptr;
01568   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01569     return SDValue(E, 0);
01570 
01571   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01572   // SDNode doesn't have access to it.  This memory will be "leaked" when
01573   // the node is deallocated, but recovered when the NodeAllocator is released.
01574   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01575   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01576 
01577   ShuffleVectorSDNode *N =
01578     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01579                                             dl.getDebugLoc(), N1, N2,
01580                                             MaskAlloc);
01581   CSEMap.InsertNode(N, IP);
01582   InsertNode(N);
01583   return SDValue(N, 0);
01584 }
01585 
01586 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01587   MVT VT = SV.getSimpleValueType(0);
01588   unsigned NumElems = VT.getVectorNumElements();
01589   SmallVector<int, 8> MaskVec;
01590 
01591   for (unsigned i = 0; i != NumElems; ++i) {
01592     int Idx = SV.getMaskElt(i);
01593     if (Idx >= 0) {
01594       if (Idx < (int)NumElems)
01595         Idx += NumElems;
01596       else
01597         Idx -= NumElems;
01598     }
01599     MaskVec.push_back(Idx);
01600   }
01601 
01602   SDValue Op0 = SV.getOperand(0);
01603   SDValue Op1 = SV.getOperand(1);
01604   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01605 }
01606 
01607 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01608                                        SDValue Val, SDValue DTy,
01609                                        SDValue STy, SDValue Rnd, SDValue Sat,
01610                                        ISD::CvtCode Code) {
01611   // If the src and dest types are the same and the conversion is between
01612   // integer types of the same sign or two floats, no conversion is necessary.
01613   if (DTy == STy &&
01614       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01615     return Val;
01616 
01617   FoldingSetNodeID ID;
01618   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01619   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01620   void* IP = nullptr;
01621   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01622     return SDValue(E, 0);
01623 
01624   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01625                                                            dl.getDebugLoc(),
01626                                                            Ops, Code);
01627   CSEMap.InsertNode(N, IP);
01628   InsertNode(N);
01629   return SDValue(N, 0);
01630 }
01631 
01632 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01633   FoldingSetNodeID ID;
01634   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01635   ID.AddInteger(RegNo);
01636   void *IP = nullptr;
01637   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01638     return SDValue(E, 0);
01639 
01640   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01641   CSEMap.InsertNode(N, IP);
01642   InsertNode(N);
01643   return SDValue(N, 0);
01644 }
01645 
01646 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01647   FoldingSetNodeID ID;
01648   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01649   ID.AddPointer(RegMask);
01650   void *IP = nullptr;
01651   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01652     return SDValue(E, 0);
01653 
01654   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01655   CSEMap.InsertNode(N, IP);
01656   InsertNode(N);
01657   return SDValue(N, 0);
01658 }
01659 
01660 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01661   FoldingSetNodeID ID;
01662   SDValue Ops[] = { Root };
01663   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01664   ID.AddPointer(Label);
01665   void *IP = nullptr;
01666   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01667     return SDValue(E, 0);
01668 
01669   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01670                                                 dl.getDebugLoc(), Root, Label);
01671   CSEMap.InsertNode(N, IP);
01672   InsertNode(N);
01673   return SDValue(N, 0);
01674 }
01675 
01676 
01677 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01678                                       int64_t Offset,
01679                                       bool isTarget,
01680                                       unsigned char TargetFlags) {
01681   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01682 
01683   FoldingSetNodeID ID;
01684   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01685   ID.AddPointer(BA);
01686   ID.AddInteger(Offset);
01687   ID.AddInteger(TargetFlags);
01688   void *IP = nullptr;
01689   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01690     return SDValue(E, 0);
01691 
01692   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01693                                                      TargetFlags);
01694   CSEMap.InsertNode(N, IP);
01695   InsertNode(N);
01696   return SDValue(N, 0);
01697 }
01698 
01699 SDValue SelectionDAG::getSrcValue(const Value *V) {
01700   assert((!V || V->getType()->isPointerTy()) &&
01701          "SrcValue is not a pointer?");
01702 
01703   FoldingSetNodeID ID;
01704   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01705   ID.AddPointer(V);
01706 
01707   void *IP = nullptr;
01708   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01709     return SDValue(E, 0);
01710 
01711   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01712   CSEMap.InsertNode(N, IP);
01713   InsertNode(N);
01714   return SDValue(N, 0);
01715 }
01716 
01717 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01718 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01719   FoldingSetNodeID ID;
01720   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01721   ID.AddPointer(MD);
01722 
01723   void *IP = nullptr;
01724   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01725     return SDValue(E, 0);
01726 
01727   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01728   CSEMap.InsertNode(N, IP);
01729   InsertNode(N);
01730   return SDValue(N, 0);
01731 }
01732 
01733 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01734 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01735                                        unsigned SrcAS, unsigned DestAS) {
01736   SDValue Ops[] = {Ptr};
01737   FoldingSetNodeID ID;
01738   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01739   ID.AddInteger(SrcAS);
01740   ID.AddInteger(DestAS);
01741 
01742   void *IP = nullptr;
01743   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01744     return SDValue(E, 0);
01745 
01746   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01747                                                       dl.getDebugLoc(),
01748                                                       VT, Ptr, SrcAS, DestAS);
01749   CSEMap.InsertNode(N, IP);
01750   InsertNode(N);
01751   return SDValue(N, 0);
01752 }
01753 
01754 /// getShiftAmountOperand - Return the specified value casted to
01755 /// the target's desired shift amount type.
01756 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01757   EVT OpTy = Op.getValueType();
01758   EVT ShTy =
01759       TM.getSubtargetImpl()->getTargetLowering()->getShiftAmountTy(LHSTy);
01760   if (OpTy == ShTy || OpTy.isVector()) return Op;
01761 
01762   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01763   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01764 }
01765 
01766 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01767 /// specified value type.
01768 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01769   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01770   unsigned ByteSize = VT.getStoreSize();
01771   Type *Ty = VT.getTypeForEVT(*getContext());
01772   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01773   unsigned StackAlign =
01774   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01775 
01776   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01777   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01778 }
01779 
01780 /// CreateStackTemporary - Create a stack temporary suitable for holding
01781 /// either of the specified value types.
01782 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01783   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01784                             VT2.getStoreSizeInBits())/8;
01785   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01786   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01787   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01788   const DataLayout *TD = TLI->getDataLayout();
01789   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01790                             TD->getPrefTypeAlignment(Ty2));
01791 
01792   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01793   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01794   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01795 }
01796 
01797 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01798                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01799   // These setcc operations always fold.
01800   switch (Cond) {
01801   default: break;
01802   case ISD::SETFALSE:
01803   case ISD::SETFALSE2: return getConstant(0, VT);
01804   case ISD::SETTRUE:
01805   case ISD::SETTRUE2: {
01806     const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01807     TargetLowering::BooleanContent Cnt =
01808         TLI->getBooleanContents(N1->getValueType(0));
01809     return getConstant(
01810         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01811   }
01812 
01813   case ISD::SETOEQ:
01814   case ISD::SETOGT:
01815   case ISD::SETOGE:
01816   case ISD::SETOLT:
01817   case ISD::SETOLE:
01818   case ISD::SETONE:
01819   case ISD::SETO:
01820   case ISD::SETUO:
01821   case ISD::SETUEQ:
01822   case ISD::SETUNE:
01823     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01824     break;
01825   }
01826 
01827   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01828     const APInt &C2 = N2C->getAPIntValue();
01829     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01830       const APInt &C1 = N1C->getAPIntValue();
01831 
01832       switch (Cond) {
01833       default: llvm_unreachable("Unknown integer setcc!");
01834       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01835       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01836       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01837       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01838       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01839       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01840       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01841       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01842       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01843       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01844       }
01845     }
01846   }
01847   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01848     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01849       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01850       switch (Cond) {
01851       default: break;
01852       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01853                           return getUNDEF(VT);
01854                         // fall through
01855       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01856       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01857                           return getUNDEF(VT);
01858                         // fall through
01859       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01860                                            R==APFloat::cmpLessThan, VT);
01861       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01862                           return getUNDEF(VT);
01863                         // fall through
01864       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01865       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01866                           return getUNDEF(VT);
01867                         // fall through
01868       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01869       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01870                           return getUNDEF(VT);
01871                         // fall through
01872       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01873                                            R==APFloat::cmpEqual, VT);
01874       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01875                           return getUNDEF(VT);
01876                         // fall through
01877       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01878                                            R==APFloat::cmpEqual, VT);
01879       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01880       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01881       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01882                                            R==APFloat::cmpEqual, VT);
01883       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01884       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01885                                            R==APFloat::cmpLessThan, VT);
01886       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01887                                            R==APFloat::cmpUnordered, VT);
01888       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01889       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01890       }
01891     } else {
01892       // Ensure that the constant occurs on the RHS.
01893       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01894       MVT CompVT = N1.getValueType().getSimpleVT();
01895       if (!TM.getSubtargetImpl()->getTargetLowering()->isCondCodeLegal(
01896               SwappedCond, CompVT))
01897         return SDValue();
01898 
01899       return getSetCC(dl, VT, N2, N1, SwappedCond);
01900     }
01901   }
01902 
01903   // Could not fold it.
01904   return SDValue();
01905 }
01906 
01907 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01908 /// use this predicate to simplify operations downstream.
01909 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01910   // This predicate is not safe for vector operations.
01911   if (Op.getValueType().isVector())
01912     return false;
01913 
01914   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01915   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01916 }
01917 
01918 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01919 /// this predicate to simplify operations downstream.  Mask is known to be zero
01920 /// for bits that V cannot have.
01921 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01922                                      unsigned Depth) const {
01923   APInt KnownZero, KnownOne;
01924   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01925   return (KnownZero & Mask) == Mask;
01926 }
01927 
01928 /// Determine which bits of Op are known to be either zero or one and return
01929 /// them in the KnownZero/KnownOne bitsets.
01930 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01931                                     APInt &KnownOne, unsigned Depth) const {
01932   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01933   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01934 
01935   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01936   if (Depth == 6)
01937     return;  // Limit search depth.
01938 
01939   APInt KnownZero2, KnownOne2;
01940 
01941   switch (Op.getOpcode()) {
01942   case ISD::Constant:
01943     // We know all of the bits for a constant!
01944     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01945     KnownZero = ~KnownOne;
01946     break;
01947   case ISD::AND:
01948     // If either the LHS or the RHS are Zero, the result is zero.
01949     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01950     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01951 
01952     // Output known-1 bits are only known if set in both the LHS & RHS.
01953     KnownOne &= KnownOne2;
01954     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01955     KnownZero |= KnownZero2;
01956     break;
01957   case ISD::OR:
01958     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01959     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01960 
01961     // Output known-0 bits are only known if clear in both the LHS & RHS.
01962     KnownZero &= KnownZero2;
01963     // Output known-1 are known to be set if set in either the LHS | RHS.
01964     KnownOne |= KnownOne2;
01965     break;
01966   case ISD::XOR: {
01967     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01968     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01969 
01970     // Output known-0 bits are known if clear or set in both the LHS & RHS.
01971     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
01972     // Output known-1 are known to be set if set in only one of the LHS, RHS.
01973     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
01974     KnownZero = KnownZeroOut;
01975     break;
01976   }
01977   case ISD::MUL: {
01978     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01979     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01980 
01981     // If low bits are zero in either operand, output low known-0 bits.
01982     // Also compute a conserative estimate for high known-0 bits.
01983     // More trickiness is possible, but this is sufficient for the
01984     // interesting case of alignment computation.
01985     KnownOne.clearAllBits();
01986     unsigned TrailZ = KnownZero.countTrailingOnes() +
01987                       KnownZero2.countTrailingOnes();
01988     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
01989                                KnownZero2.countLeadingOnes(),
01990                                BitWidth) - BitWidth;
01991 
01992     TrailZ = std::min(TrailZ, BitWidth);
01993     LeadZ = std::min(LeadZ, BitWidth);
01994     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
01995                 APInt::getHighBitsSet(BitWidth, LeadZ);
01996     break;
01997   }
01998   case ISD::UDIV: {
01999     // For the purposes of computing leading zeros we can conservatively
02000     // treat a udiv as a logical right shift by the power of 2 known to
02001     // be less than the denominator.
02002     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02003     unsigned LeadZ = KnownZero2.countLeadingOnes();
02004 
02005     KnownOne2.clearAllBits();
02006     KnownZero2.clearAllBits();
02007     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02008     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
02009     if (RHSUnknownLeadingOnes != BitWidth)
02010       LeadZ = std::min(BitWidth,
02011                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
02012 
02013     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02014     break;
02015   }
02016   case ISD::SELECT:
02017     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02018     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02019 
02020     // Only known if known in both the LHS and RHS.
02021     KnownOne &= KnownOne2;
02022     KnownZero &= KnownZero2;
02023     break;
02024   case ISD::SELECT_CC:
02025     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02026     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02027 
02028     // Only known if known in both the LHS and RHS.
02029     KnownOne &= KnownOne2;
02030     KnownZero &= KnownZero2;
02031     break;
02032   case ISD::SADDO:
02033   case ISD::UADDO:
02034   case ISD::SSUBO:
02035   case ISD::USUBO:
02036   case ISD::SMULO:
02037   case ISD::UMULO:
02038     if (Op.getResNo() != 1)
02039       break;
02040     // The boolean result conforms to getBooleanContents.
02041     // If we know the result of a setcc has the top bits zero, use this info.
02042     // We know that we have an integer-based boolean since these operations
02043     // are only available for integer.
02044     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02045             TargetLowering::ZeroOrOneBooleanContent &&
02046         BitWidth > 1)
02047       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02048     break;
02049   case ISD::SETCC:
02050     // If we know the result of a setcc has the top bits zero, use this info.
02051     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02052             TargetLowering::ZeroOrOneBooleanContent &&
02053         BitWidth > 1)
02054       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02055     break;
02056   case ISD::SHL:
02057     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02058     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02059       unsigned ShAmt = SA->getZExtValue();
02060 
02061       // If the shift count is an invalid immediate, don't do anything.
02062       if (ShAmt >= BitWidth)
02063         break;
02064 
02065       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02066       KnownZero <<= ShAmt;
02067       KnownOne  <<= ShAmt;
02068       // low bits known zero.
02069       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02070     }
02071     break;
02072   case ISD::SRL:
02073     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02074     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02075       unsigned ShAmt = SA->getZExtValue();
02076 
02077       // If the shift count is an invalid immediate, don't do anything.
02078       if (ShAmt >= BitWidth)
02079         break;
02080 
02081       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02082       KnownZero = KnownZero.lshr(ShAmt);
02083       KnownOne  = KnownOne.lshr(ShAmt);
02084 
02085       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02086       KnownZero |= HighBits;  // High bits known zero.
02087     }
02088     break;
02089   case ISD::SRA:
02090     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02091       unsigned ShAmt = SA->getZExtValue();
02092 
02093       // If the shift count is an invalid immediate, don't do anything.
02094       if (ShAmt >= BitWidth)
02095         break;
02096 
02097       // If any of the demanded bits are produced by the sign extension, we also
02098       // demand the input sign bit.
02099       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02100 
02101       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02102       KnownZero = KnownZero.lshr(ShAmt);
02103       KnownOne  = KnownOne.lshr(ShAmt);
02104 
02105       // Handle the sign bits.
02106       APInt SignBit = APInt::getSignBit(BitWidth);
02107       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02108 
02109       if (KnownZero.intersects(SignBit)) {
02110         KnownZero |= HighBits;  // New bits are known zero.
02111       } else if (KnownOne.intersects(SignBit)) {
02112         KnownOne  |= HighBits;  // New bits are known one.
02113       }
02114     }
02115     break;
02116   case ISD::SIGN_EXTEND_INREG: {
02117     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02118     unsigned EBits = EVT.getScalarType().getSizeInBits();
02119 
02120     // Sign extension.  Compute the demanded bits in the result that are not
02121     // present in the input.
02122     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02123 
02124     APInt InSignBit = APInt::getSignBit(EBits);
02125     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02126 
02127     // If the sign extended bits are demanded, we know that the sign
02128     // bit is demanded.
02129     InSignBit = InSignBit.zext(BitWidth);
02130     if (NewBits.getBoolValue())
02131       InputDemandedBits |= InSignBit;
02132 
02133     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02134     KnownOne &= InputDemandedBits;
02135     KnownZero &= InputDemandedBits;
02136 
02137     // If the sign bit of the input is known set or clear, then we know the
02138     // top bits of the result.
02139     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02140       KnownZero |= NewBits;
02141       KnownOne  &= ~NewBits;
02142     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02143       KnownOne  |= NewBits;
02144       KnownZero &= ~NewBits;
02145     } else {                              // Input sign bit unknown
02146       KnownZero &= ~NewBits;
02147       KnownOne  &= ~NewBits;
02148     }
02149     break;
02150   }
02151   case ISD::CTTZ:
02152   case ISD::CTTZ_ZERO_UNDEF:
02153   case ISD::CTLZ:
02154   case ISD::CTLZ_ZERO_UNDEF:
02155   case ISD::CTPOP: {
02156     unsigned LowBits = Log2_32(BitWidth)+1;
02157     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02158     KnownOne.clearAllBits();
02159     break;
02160   }
02161   case ISD::LOAD: {
02162     LoadSDNode *LD = cast<LoadSDNode>(Op);
02163     // If this is a ZEXTLoad and we are looking at the loaded value.
02164     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02165       EVT VT = LD->getMemoryVT();
02166       unsigned MemBits = VT.getScalarType().getSizeInBits();
02167       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02168     } else if (const MDNode *Ranges = LD->getRanges()) {
02169       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02170     }
02171     break;
02172   }
02173   case ISD::ZERO_EXTEND: {
02174     EVT InVT = Op.getOperand(0).getValueType();
02175     unsigned InBits = InVT.getScalarType().getSizeInBits();
02176     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02177     KnownZero = KnownZero.trunc(InBits);
02178     KnownOne = KnownOne.trunc(InBits);
02179     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02180     KnownZero = KnownZero.zext(BitWidth);
02181     KnownOne = KnownOne.zext(BitWidth);
02182     KnownZero |= NewBits;
02183     break;
02184   }
02185   case ISD::SIGN_EXTEND: {
02186     EVT InVT = Op.getOperand(0).getValueType();
02187     unsigned InBits = InVT.getScalarType().getSizeInBits();
02188     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02189 
02190     KnownZero = KnownZero.trunc(InBits);
02191     KnownOne = KnownOne.trunc(InBits);
02192     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02193 
02194     // Note if the sign bit is known to be zero or one.
02195     bool SignBitKnownZero = KnownZero.isNegative();
02196     bool SignBitKnownOne  = KnownOne.isNegative();
02197 
02198     KnownZero = KnownZero.zext(BitWidth);
02199     KnownOne = KnownOne.zext(BitWidth);
02200 
02201     // If the sign bit is known zero or one, the top bits match.
02202     if (SignBitKnownZero)
02203       KnownZero |= NewBits;
02204     else if (SignBitKnownOne)
02205       KnownOne  |= NewBits;
02206     break;
02207   }
02208   case ISD::ANY_EXTEND: {
02209     EVT InVT = Op.getOperand(0).getValueType();
02210     unsigned InBits = InVT.getScalarType().getSizeInBits();
02211     KnownZero = KnownZero.trunc(InBits);
02212     KnownOne = KnownOne.trunc(InBits);
02213     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02214     KnownZero = KnownZero.zext(BitWidth);
02215     KnownOne = KnownOne.zext(BitWidth);
02216     break;
02217   }
02218   case ISD::TRUNCATE: {
02219     EVT InVT = Op.getOperand(0).getValueType();
02220     unsigned InBits = InVT.getScalarType().getSizeInBits();
02221     KnownZero = KnownZero.zext(InBits);
02222     KnownOne = KnownOne.zext(InBits);
02223     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02224     KnownZero = KnownZero.trunc(BitWidth);
02225     KnownOne = KnownOne.trunc(BitWidth);
02226     break;
02227   }
02228   case ISD::AssertZext: {
02229     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02230     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02231     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02232     KnownZero |= (~InMask);
02233     KnownOne  &= (~KnownZero);
02234     break;
02235   }
02236   case ISD::FGETSIGN:
02237     // All bits are zero except the low bit.
02238     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02239     break;
02240 
02241   case ISD::SUB: {
02242     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02243       // We know that the top bits of C-X are clear if X contains less bits
02244       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02245       // positive if we can prove that X is >= 0 and < 16.
02246       if (CLHS->getAPIntValue().isNonNegative()) {
02247         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02248         // NLZ can't be BitWidth with no sign bit
02249         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02250         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02251 
02252         // If all of the MaskV bits are known to be zero, then we know the
02253         // output top bits are zero, because we now know that the output is
02254         // from [0-C].
02255         if ((KnownZero2 & MaskV) == MaskV) {
02256           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02257           // Top bits known zero.
02258           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02259         }
02260       }
02261     }
02262   }
02263   // fall through
02264   case ISD::ADD:
02265   case ISD::ADDE: {
02266     // Output known-0 bits are known if clear or set in both the low clear bits
02267     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02268     // low 3 bits clear.
02269     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02270     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02271 
02272     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02273     KnownZeroOut = std::min(KnownZeroOut,
02274                             KnownZero2.countTrailingOnes());
02275 
02276     if (Op.getOpcode() == ISD::ADD) {
02277       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02278       break;
02279     }
02280 
02281     // With ADDE, a carry bit may be added in, so we can only use this
02282     // information if we know (at least) that the low two bits are clear.  We
02283     // then return to the caller that the low bit is unknown but that other bits
02284     // are known zero.
02285     if (KnownZeroOut >= 2) // ADDE
02286       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02287     break;
02288   }
02289   case ISD::SREM:
02290     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02291       const APInt &RA = Rem->getAPIntValue().abs();
02292       if (RA.isPowerOf2()) {
02293         APInt LowBits = RA - 1;
02294         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02295 
02296         // The low bits of the first operand are unchanged by the srem.
02297         KnownZero = KnownZero2 & LowBits;
02298         KnownOne = KnownOne2 & LowBits;
02299 
02300         // If the first operand is non-negative or has all low bits zero, then
02301         // the upper bits are all zero.
02302         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02303           KnownZero |= ~LowBits;
02304 
02305         // If the first operand is negative and not all low bits are zero, then
02306         // the upper bits are all one.
02307         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02308           KnownOne |= ~LowBits;
02309         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02310       }
02311     }
02312     break;
02313   case ISD::UREM: {
02314     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02315       const APInt &RA = Rem->getAPIntValue();
02316       if (RA.isPowerOf2()) {
02317         APInt LowBits = (RA - 1);
02318         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02319 
02320         // The upper bits are all zero, the lower ones are unchanged.
02321         KnownZero = KnownZero2 | ~LowBits;
02322         KnownOne = KnownOne2 & LowBits;
02323         break;
02324       }
02325     }
02326 
02327     // Since the result is less than or equal to either operand, any leading
02328     // zero bits in either operand must also exist in the result.
02329     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02330     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02331 
02332     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02333                                 KnownZero2.countLeadingOnes());
02334     KnownOne.clearAllBits();
02335     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02336     break;
02337   }
02338   case ISD::FrameIndex:
02339   case ISD::TargetFrameIndex:
02340     if (unsigned Align = InferPtrAlignment(Op)) {
02341       // The low bits are known zero if the pointer is aligned.
02342       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02343       break;
02344     }
02345     break;
02346 
02347   default:
02348     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02349       break;
02350     // Fallthrough
02351   case ISD::INTRINSIC_WO_CHAIN:
02352   case ISD::INTRINSIC_W_CHAIN:
02353   case ISD::INTRINSIC_VOID:
02354     // Allow the target to implement this method for its nodes.
02355     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02356     break;
02357   }
02358 
02359   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02360 }
02361 
02362 /// ComputeNumSignBits - Return the number of times the sign bit of the
02363 /// register is replicated into the other bits.  We know that at least 1 bit
02364 /// is always equal to the sign bit (itself), but other cases can give us
02365 /// information.  For example, immediately after an "SRA X, 2", we know that
02366 /// the top 3 bits are all equal to each other, so we return 3.
02367 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02368   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02369   EVT VT = Op.getValueType();
02370   assert(VT.isInteger() && "Invalid VT!");
02371   unsigned VTBits = VT.getScalarType().getSizeInBits();
02372   unsigned Tmp, Tmp2;
02373   unsigned FirstAnswer = 1;
02374 
02375   if (Depth == 6)
02376     return 1;  // Limit search depth.
02377 
02378   switch (Op.getOpcode()) {
02379   default: break;
02380   case ISD::AssertSext:
02381     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02382     return VTBits-Tmp+1;
02383   case ISD::AssertZext:
02384     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02385     return VTBits-Tmp;
02386 
02387   case ISD::Constant: {
02388     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02389     return Val.getNumSignBits();
02390   }
02391 
02392   case ISD::SIGN_EXTEND:
02393     Tmp =
02394         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02395     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02396 
02397   case ISD::SIGN_EXTEND_INREG:
02398     // Max of the input and what this extends.
02399     Tmp =
02400       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02401     Tmp = VTBits-Tmp+1;
02402 
02403     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02404     return std::max(Tmp, Tmp2);
02405 
02406   case ISD::SRA:
02407     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02408     // SRA X, C   -> adds C sign bits.
02409     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02410       Tmp += C->getZExtValue();
02411       if (Tmp > VTBits) Tmp = VTBits;
02412     }
02413     return Tmp;
02414   case ISD::SHL:
02415     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02416       // shl destroys sign bits.
02417       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02418       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02419           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02420       return Tmp - C->getZExtValue();
02421     }
02422     break;
02423   case ISD::AND:
02424   case ISD::OR:
02425   case ISD::XOR:    // NOT is handled here.
02426     // Logical binary ops preserve the number of sign bits at the worst.
02427     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02428     if (Tmp != 1) {
02429       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02430       FirstAnswer = std::min(Tmp, Tmp2);
02431       // We computed what we know about the sign bits as our first
02432       // answer. Now proceed to the generic code that uses
02433       // computeKnownBits, and pick whichever answer is better.
02434     }
02435     break;
02436 
02437   case ISD::SELECT:
02438     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02439     if (Tmp == 1) return 1;  // Early out.
02440     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02441     return std::min(Tmp, Tmp2);
02442 
02443   case ISD::SADDO:
02444   case ISD::UADDO:
02445   case ISD::SSUBO:
02446   case ISD::USUBO:
02447   case ISD::SMULO:
02448   case ISD::UMULO:
02449     if (Op.getResNo() != 1)
02450       break;
02451     // The boolean result conforms to getBooleanContents.  Fall through.
02452     // If setcc returns 0/-1, all bits are sign bits.
02453     // We know that we have an integer-based boolean since these operations
02454     // are only available for integer.
02455     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02456         TargetLowering::ZeroOrNegativeOneBooleanContent)
02457       return VTBits;
02458     break;
02459   case ISD::SETCC:
02460     // If setcc returns 0/-1, all bits are sign bits.
02461     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02462         TargetLowering::ZeroOrNegativeOneBooleanContent)
02463       return VTBits;
02464     break;
02465   case ISD::ROTL:
02466   case ISD::ROTR:
02467     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02468       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02469 
02470       // Handle rotate right by N like a rotate left by 32-N.
02471       if (Op.getOpcode() == ISD::ROTR)
02472         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02473 
02474       // If we aren't rotating out all of the known-in sign bits, return the
02475       // number that are left.  This handles rotl(sext(x), 1) for example.
02476       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02477       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02478     }
02479     break;
02480   case ISD::ADD:
02481     // Add can have at most one carry bit.  Thus we know that the output
02482     // is, at worst, one more bit than the inputs.
02483     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02484     if (Tmp == 1) return 1;  // Early out.
02485 
02486     // Special case decrementing a value (ADD X, -1):
02487     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02488       if (CRHS->isAllOnesValue()) {
02489         APInt KnownZero, KnownOne;
02490         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02491 
02492         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02493         // sign bits set.
02494         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02495           return VTBits;
02496 
02497         // If we are subtracting one from a positive number, there is no carry
02498         // out of the result.
02499         if (KnownZero.isNegative())
02500           return Tmp;
02501       }
02502 
02503     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02504     if (Tmp2 == 1) return 1;
02505     return std::min(Tmp, Tmp2)-1;
02506 
02507   case ISD::SUB:
02508     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02509     if (Tmp2 == 1) return 1;
02510 
02511     // Handle NEG.
02512     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02513       if (CLHS->isNullValue()) {
02514         APInt KnownZero, KnownOne;
02515         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02516         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02517         // sign bits set.
02518         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02519           return VTBits;
02520 
02521         // If the input is known to be positive (the sign bit is known clear),
02522         // the output of the NEG has the same number of sign bits as the input.
02523         if (KnownZero.isNegative())
02524           return Tmp2;
02525 
02526         // Otherwise, we treat this like a SUB.
02527       }
02528 
02529     // Sub can have at most one carry bit.  Thus we know that the output
02530     // is, at worst, one more bit than the inputs.
02531     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02532     if (Tmp == 1) return 1;  // Early out.
02533     return std::min(Tmp, Tmp2)-1;
02534   case ISD::TRUNCATE:
02535     // FIXME: it's tricky to do anything useful for this, but it is an important
02536     // case for targets like X86.
02537     break;
02538   }
02539 
02540   // If we are looking at the loaded value of the SDNode.
02541   if (Op.getResNo() == 0) {
02542     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02543     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02544       unsigned ExtType = LD->getExtensionType();
02545       switch (ExtType) {
02546         default: break;
02547         case ISD::SEXTLOAD:    // '17' bits known
02548           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02549           return VTBits-Tmp+1;
02550         case ISD::ZEXTLOAD:    // '16' bits known
02551           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02552           return VTBits-Tmp;
02553       }
02554     }
02555   }
02556 
02557   // Allow the target to implement this method for its nodes.
02558   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02559       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02560       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02561       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02562     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02563     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02564   }
02565 
02566   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02567   // use this information.
02568   APInt KnownZero, KnownOne;
02569   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02570 
02571   APInt Mask;
02572   if (KnownZero.isNegative()) {        // sign bit is 0
02573     Mask = KnownZero;
02574   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02575     Mask = KnownOne;
02576   } else {
02577     // Nothing known.
02578     return FirstAnswer;
02579   }
02580 
02581   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02582   // the number of identical bits in the top of the input value.
02583   Mask = ~Mask;
02584   Mask <<= Mask.getBitWidth()-VTBits;
02585   // Return # leading zeros.  We use 'min' here in case Val was zero before
02586   // shifting.  We don't want to return '64' as for an i32 "0".
02587   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02588 }
02589 
02590 /// isBaseWithConstantOffset - Return true if the specified operand is an
02591 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02592 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02593 /// semantics as an ADD.  This handles the equivalence:
02594 ///     X|Cst == X+Cst iff X&Cst = 0.
02595 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02596   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02597       !isa<ConstantSDNode>(Op.getOperand(1)))
02598     return false;
02599 
02600   if (Op.getOpcode() == ISD::OR &&
02601       !MaskedValueIsZero(Op.getOperand(0),
02602                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02603     return false;
02604 
02605   return true;
02606 }
02607 
02608 
02609 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02610   // If we're told that NaNs won't happen, assume they won't.
02611   if (getTarget().Options.NoNaNsFPMath)
02612     return true;
02613 
02614   // If the value is a constant, we can obviously see if it is a NaN or not.
02615   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02616     return !C->getValueAPF().isNaN();
02617 
02618   // TODO: Recognize more cases here.
02619 
02620   return false;
02621 }
02622 
02623 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02624   // If the value is a constant, we can obviously see if it is a zero or not.
02625   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02626     return !C->isZero();
02627 
02628   // TODO: Recognize more cases here.
02629   switch (Op.getOpcode()) {
02630   default: break;
02631   case ISD::OR:
02632     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02633       return !C->isNullValue();
02634     break;
02635   }
02636 
02637   return false;
02638 }
02639 
02640 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02641   // Check the obvious case.
02642   if (A == B) return true;
02643 
02644   // For for negative and positive zero.
02645   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02646     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02647       if (CA->isZero() && CB->isZero()) return true;
02648 
02649   // Otherwise they may not be equal.
02650   return false;
02651 }
02652 
02653 /// getNode - Gets or creates the specified node.
02654 ///
02655 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02656   FoldingSetNodeID ID;
02657   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02658   void *IP = nullptr;
02659   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02660     return SDValue(E, 0);
02661 
02662   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02663                                          DL.getDebugLoc(), getVTList(VT));
02664   CSEMap.InsertNode(N, IP);
02665 
02666   InsertNode(N);
02667   return SDValue(N, 0);
02668 }
02669 
02670 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02671                               EVT VT, SDValue Operand) {
02672   // Constant fold unary operations with an integer constant operand. Even
02673   // opaque constant will be folded, because the folding of unary operations
02674   // doesn't create new constants with different values. Nevertheless, the
02675   // opaque flag is preserved during folding to prevent future folding with
02676   // other constants.
02677   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02678     const APInt &Val = C->getAPIntValue();
02679     switch (Opcode) {
02680     default: break;
02681     case ISD::SIGN_EXTEND:
02682       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02683                          C->isTargetOpcode(), C->isOpaque());
02684     case ISD::ANY_EXTEND:
02685     case ISD::ZERO_EXTEND:
02686     case ISD::TRUNCATE:
02687       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02688                          C->isTargetOpcode(), C->isOpaque());
02689     case ISD::UINT_TO_FP:
02690     case ISD::SINT_TO_FP: {
02691       APFloat apf(EVTToAPFloatSemantics(VT),
02692                   APInt::getNullValue(VT.getSizeInBits()));
02693       (void)apf.convertFromAPInt(Val,
02694                                  Opcode==ISD::SINT_TO_FP,
02695                                  APFloat::rmNearestTiesToEven);
02696       return getConstantFP(apf, VT);
02697     }
02698     case ISD::BITCAST:
02699       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02700         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02701       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02702         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02703       break;
02704     case ISD::BSWAP:
02705       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02706                          C->isOpaque());
02707     case ISD::CTPOP:
02708       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02709                          C->isOpaque());
02710     case ISD::CTLZ:
02711     case ISD::CTLZ_ZERO_UNDEF:
02712       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02713                          C->isOpaque());
02714     case ISD::CTTZ:
02715     case ISD::CTTZ_ZERO_UNDEF:
02716       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02717                          C->isOpaque());
02718     }
02719   }
02720 
02721   // Constant fold unary operations with a floating point constant operand.
02722   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02723     APFloat V = C->getValueAPF();    // make copy
02724     switch (Opcode) {
02725     case ISD::FNEG:
02726       V.changeSign();
02727       return getConstantFP(V, VT);
02728     case ISD::FABS:
02729       V.clearSign();
02730       return getConstantFP(V, VT);
02731     case ISD::FCEIL: {
02732       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02733       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02734         return getConstantFP(V, VT);
02735       break;
02736     }
02737     case ISD::FTRUNC: {
02738       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02739       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02740         return getConstantFP(V, VT);
02741       break;
02742     }
02743     case ISD::FFLOOR: {
02744       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02745       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02746         return getConstantFP(V, VT);
02747       break;
02748     }
02749     case ISD::FP_EXTEND: {
02750       bool ignored;
02751       // This can return overflow, underflow, or inexact; we don't care.
02752       // FIXME need to be more flexible about rounding mode.
02753       (void)V.convert(EVTToAPFloatSemantics(VT),
02754                       APFloat::rmNearestTiesToEven, &ignored);
02755       return getConstantFP(V, VT);
02756     }
02757     case ISD::FP_TO_SINT:
02758     case ISD::FP_TO_UINT: {
02759       integerPart x[2];
02760       bool ignored;
02761       assert(integerPartWidth >= 64);
02762       // FIXME need to be more flexible about rounding mode.
02763       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02764                             Opcode==ISD::FP_TO_SINT,
02765                             APFloat::rmTowardZero, &ignored);
02766       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02767         break;
02768       APInt api(VT.getSizeInBits(), x);
02769       return getConstant(api, VT);
02770     }
02771     case ISD::BITCAST:
02772       if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02773         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02774       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02775         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02776       break;
02777     }
02778   }
02779 
02780   // Constant fold unary operations with a vector integer operand.
02781   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02782     if (BV->isConstant()) {
02783       switch (Opcode) {
02784       default:
02785         // FIXME: Entirely reasonable to perform folding of other unary
02786         // operations here as the need arises.
02787         break;
02788       case ISD::UINT_TO_FP:
02789       case ISD::SINT_TO_FP: {
02790         SmallVector<SDValue, 8> Ops;
02791         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02792           SDValue OpN = BV->getOperand(i);
02793           // Let the above scalar folding handle the conversion of each
02794           // element.
02795           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02796                         OpN);
02797           Ops.push_back(OpN);
02798         }
02799         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02800       }
02801       }
02802     }
02803   }
02804 
02805   unsigned OpOpcode = Operand.getNode()->getOpcode();
02806   switch (Opcode) {
02807   case ISD::TokenFactor:
02808   case ISD::MERGE_VALUES:
02809   case ISD::CONCAT_VECTORS:
02810     return Operand;         // Factor, merge or concat of one node?  No need.
02811   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02812   case ISD::FP_EXTEND:
02813     assert(VT.isFloatingPoint() &&
02814            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02815     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02816     assert((!VT.isVector() ||
02817             VT.getVectorNumElements() ==
02818             Operand.getValueType().getVectorNumElements()) &&
02819            "Vector element count mismatch!");
02820     if (Operand.getOpcode() == ISD::UNDEF)
02821       return getUNDEF(VT);
02822     break;
02823   case ISD::SIGN_EXTEND:
02824     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02825            "Invalid SIGN_EXTEND!");
02826     if (Operand.getValueType() == VT) return Operand;   // noop extension
02827     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02828            "Invalid sext node, dst < src!");
02829     assert((!VT.isVector() ||
02830             VT.getVectorNumElements() ==
02831             Operand.getValueType().getVectorNumElements()) &&
02832            "Vector element count mismatch!");
02833     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02834       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02835     else if (OpOpcode == ISD::UNDEF)
02836       // sext(undef) = 0, because the top bits will all be the same.
02837       return getConstant(0, VT);
02838     break;
02839   case ISD::ZERO_EXTEND:
02840     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02841            "Invalid ZERO_EXTEND!");
02842     if (Operand.getValueType() == VT) return Operand;   // noop extension
02843     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02844            "Invalid zext node, dst < src!");
02845     assert((!VT.isVector() ||
02846             VT.getVectorNumElements() ==
02847             Operand.getValueType().getVectorNumElements()) &&
02848            "Vector element count mismatch!");
02849     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02850       return getNode(ISD::ZERO_EXTEND, DL, VT,
02851                      Operand.getNode()->getOperand(0));
02852     else if (OpOpcode == ISD::UNDEF)
02853       // zext(undef) = 0, because the top bits will be zero.
02854       return getConstant(0, VT);
02855     break;
02856   case ISD::ANY_EXTEND:
02857     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02858            "Invalid ANY_EXTEND!");
02859     if (Operand.getValueType() == VT) return Operand;   // noop extension
02860     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02861            "Invalid anyext node, dst < src!");
02862     assert((!VT.isVector() ||
02863             VT.getVectorNumElements() ==
02864             Operand.getValueType().getVectorNumElements()) &&
02865            "Vector element count mismatch!");
02866 
02867     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02868         OpOpcode == ISD::ANY_EXTEND)
02869       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02870       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02871     else if (OpOpcode == ISD::UNDEF)
02872       return getUNDEF(VT);
02873 
02874     // (ext (trunx x)) -> x
02875     if (OpOpcode == ISD::TRUNCATE) {
02876       SDValue OpOp = Operand.getNode()->getOperand(0);
02877       if (OpOp.getValueType() == VT)
02878         return OpOp;
02879     }
02880     break;
02881   case ISD::TRUNCATE:
02882     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02883            "Invalid TRUNCATE!");
02884     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02885     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02886            "Invalid truncate node, src < dst!");
02887     assert((!VT.isVector() ||
02888             VT.getVectorNumElements() ==
02889             Operand.getValueType().getVectorNumElements()) &&
02890            "Vector element count mismatch!");
02891     if (OpOpcode == ISD::TRUNCATE)
02892       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02893     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02894         OpOpcode == ISD::ANY_EXTEND) {
02895       // If the source is smaller than the dest, we still need an extend.
02896       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02897             .bitsLT(VT.getScalarType()))
02898         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02899       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02900         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02901       return Operand.getNode()->getOperand(0);
02902     }
02903     if (OpOpcode == ISD::UNDEF)
02904       return getUNDEF(VT);
02905     break;
02906   case ISD::BITCAST:
02907     // Basic sanity checking.
02908     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02909            && "Cannot BITCAST between types of different sizes!");
02910     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02911     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02912       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02913     if (OpOpcode == ISD::UNDEF)
02914       return getUNDEF(VT);
02915     break;
02916   case ISD::SCALAR_TO_VECTOR:
02917     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02918            (VT.getVectorElementType() == Operand.getValueType() ||
02919             (VT.getVectorElementType().isInteger() &&
02920              Operand.getValueType().isInteger() &&
02921              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02922            "Illegal SCALAR_TO_VECTOR node!");
02923     if (OpOpcode == ISD::UNDEF)
02924       return getUNDEF(VT);
02925     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02926     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02927         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02928         Operand.getConstantOperandVal(1) == 0 &&
02929         Operand.getOperand(0).getValueType() == VT)
02930       return Operand.getOperand(0);
02931     break;
02932   case ISD::FNEG:
02933     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
02934     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
02935       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
02936                      Operand.getNode()->getOperand(0));
02937     if (OpOpcode == ISD::FNEG)  // --X -> X
02938       return Operand.getNode()->getOperand(0);
02939     break;
02940   case ISD::FABS:
02941     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
02942       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
02943     break;
02944   }
02945 
02946   SDNode *N;
02947   SDVTList VTs = getVTList(VT);
02948   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
02949     FoldingSetNodeID ID;
02950     SDValue Ops[1] = { Operand };
02951     AddNodeIDNode(ID, Opcode, VTs, Ops);
02952     void *IP = nullptr;
02953     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02954       return SDValue(E, 0);
02955 
02956     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02957                                         DL.getDebugLoc(), VTs, Operand);
02958     CSEMap.InsertNode(N, IP);
02959   } else {
02960     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02961                                         DL.getDebugLoc(), VTs, Operand);
02962   }
02963 
02964   InsertNode(N);
02965   return SDValue(N, 0);
02966 }
02967 
02968 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
02969                                              SDNode *Cst1, SDNode *Cst2) {
02970   // If the opcode is a target-specific ISD node, there's nothing we can
02971   // do here and the operand rules may not line up with the below, so
02972   // bail early.
02973   if (Opcode >= ISD::BUILTIN_OP_END)
02974     return SDValue();
02975 
02976   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
02977   SmallVector<SDValue, 4> Outputs;
02978   EVT SVT = VT.getScalarType();
02979 
02980   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
02981   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
02982   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
02983     return SDValue();
02984 
02985   if (Scalar1 && Scalar2)
02986     // Scalar instruction.
02987     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
02988   else {
02989     // For vectors extract each constant element into Inputs so we can constant
02990     // fold them individually.
02991     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
02992     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
02993     if (!BV1 || !BV2)
02994       return SDValue();
02995 
02996     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
02997 
02998     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
02999       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
03000       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
03001       if (!V1 || !V2) // Not a constant, bail.
03002         return SDValue();
03003 
03004       if (V1->isOpaque() || V2->isOpaque())
03005         return SDValue();
03006 
03007       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
03008       // FIXME: This is valid and could be handled by truncating the APInts.
03009       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
03010         return SDValue();
03011 
03012       Inputs.push_back(std::make_pair(V1, V2));
03013     }
03014   }
03015 
03016   // We have a number of constant values, constant fold them element by element.
03017   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03018     const APInt &C1 = Inputs[I].first->getAPIntValue();
03019     const APInt &C2 = Inputs[I].second->getAPIntValue();
03020 
03021     switch (Opcode) {
03022     case ISD::ADD:
03023       Outputs.push_back(getConstant(C1 + C2, SVT));
03024       break;
03025     case ISD::SUB:
03026       Outputs.push_back(getConstant(C1 - C2, SVT));
03027       break;
03028     case ISD::MUL:
03029       Outputs.push_back(getConstant(C1 * C2, SVT));
03030       break;
03031     case ISD::UDIV:
03032       if (!C2.getBoolValue())
03033         return SDValue();
03034       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03035       break;
03036     case ISD::UREM:
03037       if (!C2.getBoolValue())
03038         return SDValue();
03039       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03040       break;
03041     case ISD::SDIV:
03042       if (!C2.getBoolValue())
03043         return SDValue();
03044       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03045       break;
03046     case ISD::SREM:
03047       if (!C2.getBoolValue())
03048         return SDValue();
03049       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03050       break;
03051     case ISD::AND:
03052       Outputs.push_back(getConstant(C1 & C2, SVT));
03053       break;
03054     case ISD::OR:
03055       Outputs.push_back(getConstant(C1 | C2, SVT));
03056       break;
03057     case ISD::XOR:
03058       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03059       break;
03060     case ISD::SHL:
03061       Outputs.push_back(getConstant(C1 << C2, SVT));
03062       break;
03063     case ISD::SRL:
03064       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03065       break;
03066     case ISD::SRA:
03067       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03068       break;
03069     case ISD::ROTL:
03070       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03071       break;
03072     case ISD::ROTR:
03073       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03074       break;
03075     default:
03076       return SDValue();
03077     }
03078   }
03079 
03080   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03081                                   "Expected a scalar or vector!"));
03082 
03083   // Handle the scalar case first.
03084   if (!VT.isVector())
03085     return Outputs.back();
03086 
03087   // We may have a vector type but a scalar result. Create a splat.
03088   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03089 
03090   // Build a big vector out of the scalar elements we generated.
03091   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03092 }
03093 
03094 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03095                               SDValue N2, bool nuw, bool nsw, bool exact) {
03096   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03097   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03098   switch (Opcode) {
03099   default: break;
03100   case ISD::TokenFactor:
03101     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03102            N2.getValueType() == MVT::Other && "Invalid token factor!");
03103     // Fold trivial token factors.
03104     if (N1.getOpcode() == ISD::EntryToken) return N2;
03105     if (N2.getOpcode() == ISD::EntryToken) return N1;
03106     if (N1 == N2) return N1;
03107     break;
03108   case ISD::CONCAT_VECTORS:
03109     // Concat of UNDEFs is UNDEF.
03110     if (N1.getOpcode() == ISD::UNDEF &&
03111         N2.getOpcode() == ISD::UNDEF)
03112       return getUNDEF(VT);
03113 
03114     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03115     // one big BUILD_VECTOR.
03116     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03117         N2.getOpcode() == ISD::BUILD_VECTOR) {
03118       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03119                                     N1.getNode()->op_end());
03120       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03121       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03122     }
03123     break;
03124   case ISD::AND:
03125     assert(VT.isInteger() && "This operator does not apply to FP types!");
03126     assert(N1.getValueType() == N2.getValueType() &&
03127            N1.getValueType() == VT && "Binary operator types must match!");
03128     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03129     // worth handling here.
03130     if (N2C && N2C->isNullValue())
03131       return N2;
03132     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03133       return N1;
03134     break;
03135   case ISD::OR:
03136   case ISD::XOR:
03137   case ISD::ADD:
03138   case ISD::SUB:
03139     assert(VT.isInteger() && "This operator does not apply to FP types!");
03140     assert(N1.getValueType() == N2.getValueType() &&
03141            N1.getValueType() == VT && "Binary operator types must match!");
03142     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03143     // it's worth handling here.
03144     if (N2C && N2C->isNullValue())
03145       return N1;
03146     break;
03147   case ISD::UDIV:
03148   case ISD::UREM:
03149   case ISD::MULHU:
03150   case ISD::MULHS:
03151   case ISD::MUL:
03152   case ISD::SDIV:
03153   case ISD::SREM:
03154     assert(VT.isInteger() && "This operator does not apply to FP types!");
03155     assert(N1.getValueType() == N2.getValueType() &&
03156            N1.getValueType() == VT && "Binary operator types must match!");
03157     break;
03158   case ISD::FADD:
03159   case ISD::FSUB:
03160   case ISD::FMUL:
03161   case ISD::FDIV:
03162   case ISD::FREM:
03163     if (getTarget().Options.UnsafeFPMath) {
03164       if (Opcode == ISD::FADD) {
03165         // 0+x --> x
03166         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03167           if (CFP->getValueAPF().isZero())
03168             return N2;
03169         // x+0 --> x
03170         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03171           if (CFP->getValueAPF().isZero())
03172             return N1;
03173       } else if (Opcode == ISD::FSUB) {
03174         // x-0 --> x
03175         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03176           if (CFP->getValueAPF().isZero())
03177             return N1;
03178       } else if (Opcode == ISD::FMUL) {
03179         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03180         SDValue V = N2;
03181 
03182         // If the first operand isn't the constant, try the second
03183         if (!CFP) {
03184           CFP = dyn_cast<ConstantFPSDNode>(N2);
03185           V = N1;
03186         }
03187 
03188         if (CFP) {
03189           // 0*x --> 0
03190           if (CFP->isZero())
03191             return SDValue(CFP,0);
03192           // 1*x --> x
03193           if (CFP->isExactlyValue(1.0))
03194             return V;
03195         }
03196       }
03197     }
03198     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03199     assert(N1.getValueType() == N2.getValueType() &&
03200            N1.getValueType() == VT && "Binary operator types must match!");
03201     break;
03202   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03203     assert(N1.getValueType() == VT &&
03204            N1.getValueType().isFloatingPoint() &&
03205            N2.getValueType().isFloatingPoint() &&
03206            "Invalid FCOPYSIGN!");
03207     break;
03208   case ISD::SHL:
03209   case ISD::SRA:
03210   case ISD::SRL:
03211   case ISD::ROTL:
03212   case ISD::ROTR:
03213     assert(VT == N1.getValueType() &&
03214            "Shift operators return type must be the same as their first arg");
03215     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03216            "Shifts only work on integers");
03217     assert((!VT.isVector() || VT == N2.getValueType()) &&
03218            "Vector shift amounts must be in the same as their first arg");
03219     // Verify that the shift amount VT is bit enough to hold valid shift
03220     // amounts.  This catches things like trying to shift an i1024 value by an
03221     // i8, which is easy to fall into in generic code that uses
03222     // TLI.getShiftAmount().
03223     assert(N2.getValueType().getSizeInBits() >=
03224                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03225            "Invalid use of small shift amount with oversized value!");
03226 
03227     // Always fold shifts of i1 values so the code generator doesn't need to
03228     // handle them.  Since we know the size of the shift has to be less than the
03229     // size of the value, the shift/rotate count is guaranteed to be zero.
03230     if (VT == MVT::i1)
03231       return N1;
03232     if (N2C && N2C->isNullValue())
03233       return N1;
03234     break;
03235   case ISD::FP_ROUND_INREG: {
03236     EVT EVT = cast<VTSDNode>(N2)->getVT();
03237     assert(VT == N1.getValueType() && "Not an inreg round!");
03238     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03239            "Cannot FP_ROUND_INREG integer types");
03240     assert(EVT.isVector() == VT.isVector() &&
03241            "FP_ROUND_INREG type should be vector iff the operand "
03242            "type is vector!");
03243     assert((!EVT.isVector() ||
03244             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03245            "Vector element counts must match in FP_ROUND_INREG");
03246     assert(EVT.bitsLE(VT) && "Not rounding down!");
03247     (void)EVT;
03248     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03249     break;
03250   }
03251   case ISD::FP_ROUND:
03252     assert(VT.isFloatingPoint() &&
03253            N1.getValueType().isFloatingPoint() &&
03254            VT.bitsLE(N1.getValueType()) &&
03255            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03256     if (N1.getValueType() == VT) return N1;  // noop conversion.
03257     break;
03258   case ISD::AssertSext:
03259   case ISD::AssertZext: {
03260     EVT EVT = cast<VTSDNode>(N2)->getVT();
03261     assert(VT == N1.getValueType() && "Not an inreg extend!");
03262     assert(VT.isInteger() && EVT.isInteger() &&
03263            "Cannot *_EXTEND_INREG FP types");
03264     assert(!EVT.isVector() &&
03265            "AssertSExt/AssertZExt type should be the vector element type "
03266            "rather than the vector type!");
03267     assert(EVT.bitsLE(VT) && "Not extending!");
03268     if (VT == EVT) return N1; // noop assertion.
03269     break;
03270   }
03271   case ISD::SIGN_EXTEND_INREG: {
03272     EVT EVT = cast<VTSDNode>(N2)->getVT();
03273     assert(VT == N1.getValueType() && "Not an inreg extend!");
03274     assert(VT.isInteger() && EVT.isInteger() &&
03275            "Cannot *_EXTEND_INREG FP types");
03276     assert(EVT.isVector() == VT.isVector() &&
03277            "SIGN_EXTEND_INREG type should be vector iff the operand "
03278            "type is vector!");
03279     assert((!EVT.isVector() ||
03280             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03281            "Vector element counts must match in SIGN_EXTEND_INREG");
03282     assert(EVT.bitsLE(VT) && "Not extending!");
03283     if (EVT == VT) return N1;  // Not actually extending
03284 
03285     if (N1C) {
03286       APInt Val = N1C->getAPIntValue();
03287       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03288       Val <<= Val.getBitWidth()-FromBits;
03289       Val = Val.ashr(Val.getBitWidth()-FromBits);
03290       return getConstant(Val, VT);
03291     }
03292     break;
03293   }
03294   case ISD::EXTRACT_VECTOR_ELT:
03295     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03296     if (N1.getOpcode() == ISD::UNDEF)
03297       return getUNDEF(VT);
03298 
03299     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03300     // expanding copies of large vectors from registers.
03301     if (N2C &&
03302         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03303         N1.getNumOperands() > 0) {
03304       unsigned Factor =
03305         N1.getOperand(0).getValueType().getVectorNumElements();
03306       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03307                      N1.getOperand(N2C->getZExtValue() / Factor),
03308                      getConstant(N2C->getZExtValue() % Factor,
03309                                  N2.getValueType()));
03310     }
03311 
03312     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03313     // expanding large vector constants.
03314     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03315       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03316 
03317       if (VT != Elt.getValueType())
03318         // If the vector element type is not legal, the BUILD_VECTOR operands
03319         // are promoted and implicitly truncated, and the result implicitly
03320         // extended. Make that explicit here.
03321         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03322 
03323       return Elt;
03324     }
03325 
03326     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03327     // operations are lowered to scalars.
03328     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03329       // If the indices are the same, return the inserted element else
03330       // if the indices are known different, extract the element from
03331       // the original vector.
03332       SDValue N1Op2 = N1.getOperand(2);
03333       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03334 
03335       if (N1Op2C && N2C) {
03336         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03337           if (VT == N1.getOperand(1).getValueType())
03338             return N1.getOperand(1);
03339           else
03340             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03341         }
03342 
03343         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03344       }
03345     }
03346     break;
03347   case ISD::EXTRACT_ELEMENT:
03348     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03349     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03350            (N1.getValueType().isInteger() == VT.isInteger()) &&
03351            N1.getValueType() != VT &&
03352            "Wrong types for EXTRACT_ELEMENT!");
03353 
03354     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03355     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03356     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03357     if (N1.getOpcode() == ISD::BUILD_PAIR)
03358       return N1.getOperand(N2C->getZExtValue());
03359 
03360     // EXTRACT_ELEMENT of a constant int is also very common.
03361     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03362       unsigned ElementSize = VT.getSizeInBits();
03363       unsigned Shift = ElementSize * N2C->getZExtValue();
03364       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03365       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03366     }
03367     break;
03368   case ISD::EXTRACT_SUBVECTOR: {
03369     SDValue Index = N2;
03370     if (VT.isSimple() && N1.getValueType().isSimple()) {
03371       assert(VT.isVector() && N1.getValueType().isVector() &&
03372              "Extract subvector VTs must be a vectors!");
03373       assert(VT.getVectorElementType() ==
03374              N1.getValueType().getVectorElementType() &&
03375              "Extract subvector VTs must have the same element type!");
03376       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03377              "Extract subvector must be from larger vector to smaller vector!");
03378 
03379       if (isa<ConstantSDNode>(Index.getNode())) {
03380         assert((VT.getVectorNumElements() +
03381                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03382                 <= N1.getValueType().getVectorNumElements())
03383                && "Extract subvector overflow!");
03384       }
03385 
03386       // Trivial extraction.
03387       if (VT.getSimpleVT() == N1.getSimpleValueType())
03388         return N1;
03389     }
03390     break;
03391   }
03392   }
03393 
03394   // Perform trivial constant folding.
03395   SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
03396   if (SV.getNode()) return SV;
03397 
03398   // Canonicalize constant to RHS if commutative.
03399   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03400     std::swap(N1C, N2C);
03401     std::swap(N1, N2);
03402   }
03403 
03404   // Constant fold FP operations.
03405   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03406   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03407   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03408   if (N1CFP) {
03409     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03410       // Canonicalize constant to RHS if commutative.
03411       std::swap(N1CFP, N2CFP);
03412       std::swap(N1, N2);
03413     } else if (N2CFP) {
03414       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03415       APFloat::opStatus s;
03416       switch (Opcode) {
03417       case ISD::FADD:
03418         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03419         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03420           return getConstantFP(V1, VT);
03421         break;
03422       case ISD::FSUB:
03423         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03424         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03425           return getConstantFP(V1, VT);
03426         break;
03427       case ISD::FMUL:
03428         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03429         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03430           return getConstantFP(V1, VT);
03431         break;
03432       case ISD::FDIV:
03433         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03434         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03435                                  s!=APFloat::opDivByZero)) {
03436           return getConstantFP(V1, VT);
03437         }
03438         break;
03439       case ISD::FREM :
03440         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03441         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03442                                  s!=APFloat::opDivByZero)) {
03443           return getConstantFP(V1, VT);
03444         }
03445         break;
03446       case ISD::FCOPYSIGN:
03447         V1.copySign(V2);
03448         return getConstantFP(V1, VT);
03449       default: break;
03450       }
03451     }
03452 
03453     if (Opcode == ISD::FP_ROUND) {
03454       APFloat V = N1CFP->getValueAPF();    // make copy
03455       bool ignored;
03456       // This can return overflow, underflow, or inexact; we don't care.
03457       // FIXME need to be more flexible about rounding mode.
03458       (void)V.convert(EVTToAPFloatSemantics(VT),
03459                       APFloat::rmNearestTiesToEven, &ignored);
03460       return getConstantFP(V, VT);
03461     }
03462   }
03463 
03464   // Canonicalize an UNDEF to the RHS, even over a constant.
03465   if (N1.getOpcode() == ISD::UNDEF) {
03466     if (isCommutativeBinOp(Opcode)) {
03467       std::swap(N1, N2);
03468     } else {
03469       switch (Opcode) {
03470       case ISD::FP_ROUND_INREG:
03471       case ISD::SIGN_EXTEND_INREG:
03472       case ISD::SUB:
03473       case ISD::FSUB:
03474       case ISD::FDIV:
03475       case ISD::FREM:
03476       case ISD::SRA:
03477         return N1;     // fold op(undef, arg2) -> undef
03478       case ISD::UDIV:
03479       case ISD::SDIV:
03480       case ISD::UREM:
03481       case ISD::SREM:
03482       case ISD::SRL:
03483       case ISD::SHL:
03484         if (!VT.isVector())
03485           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03486         // For vectors, we can't easily build an all zero vector, just return
03487         // the LHS.
03488         return N2;
03489       }
03490     }
03491   }
03492 
03493   // Fold a bunch of operators when the RHS is undef.
03494   if (N2.getOpcode() == ISD::UNDEF) {
03495     switch (Opcode) {
03496     case ISD::XOR:
03497       if (N1.getOpcode() == ISD::UNDEF)
03498         // Handle undef ^ undef -> 0 special case. This is a common
03499         // idiom (misuse).
03500         return getConstant(0, VT);
03501       // fallthrough
03502     case ISD::ADD:
03503     case ISD::ADDC:
03504     case ISD::ADDE:
03505     case ISD::SUB:
03506     case ISD::UDIV:
03507     case ISD::SDIV:
03508     case ISD::UREM:
03509     case ISD::SREM:
03510       return N2;       // fold op(arg1, undef) -> undef
03511     case ISD::FADD:
03512     case ISD::FSUB:
03513     case ISD::FMUL:
03514     case ISD::FDIV:
03515     case ISD::FREM:
03516       if (getTarget().Options.UnsafeFPMath)
03517         return N2;
03518       break;
03519     case ISD::MUL:
03520     case ISD::AND:
03521     case ISD::SRL:
03522     case ISD::SHL:
03523       if (!VT.isVector())
03524         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03525       // For vectors, we can't easily build an all zero vector, just return
03526       // the LHS.
03527       return N1;
03528     case ISD::OR:
03529       if (!VT.isVector())
03530         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03531       // For vectors, we can't easily build an all one vector, just return
03532       // the LHS.
03533       return N1;
03534     case ISD::SRA:
03535       return N1;
03536     }
03537   }
03538 
03539   // Memoize this node if possible.
03540   BinarySDNode *N;
03541   SDVTList VTs = getVTList(VT);
03542   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03543   if (VT != MVT::Glue) {
03544     SDValue Ops[] = {N1, N2};
03545     FoldingSetNodeID ID;
03546     AddNodeIDNode(ID, Opcode, VTs, Ops);
03547     if (BinOpHasFlags)
03548       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03549     void *IP = nullptr;
03550     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03551       return SDValue(E, 0);
03552 
03553     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03554 
03555     CSEMap.InsertNode(N, IP);
03556   } else {
03557 
03558     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03559   }
03560 
03561   InsertNode(N);
03562   return SDValue(N, 0);
03563 }
03564 
03565 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03566                               SDValue N1, SDValue N2, SDValue N3) {
03567   // Perform various simplifications.
03568   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03569   switch (Opcode) {
03570   case ISD::FMA: {
03571     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03572     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03573     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03574     if (N1CFP && N2CFP && N3CFP) {
03575       APFloat  V1 = N1CFP->getValueAPF();
03576       const APFloat &V2 = N2CFP->getValueAPF();
03577       const APFloat &V3 = N3CFP->getValueAPF();
03578       APFloat::opStatus s =
03579         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03580       if (s != APFloat::opInvalidOp)
03581         return getConstantFP(V1, VT);
03582     }
03583     break;
03584   }
03585   case ISD::CONCAT_VECTORS:
03586     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03587     // one big BUILD_VECTOR.
03588     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03589         N2.getOpcode() == ISD::BUILD_VECTOR &&
03590         N3.getOpcode() == ISD::BUILD_VECTOR) {
03591       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03592                                     N1.getNode()->op_end());
03593       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03594       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03595       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03596     }
03597     break;
03598   case ISD::SETCC: {
03599     // Use FoldSetCC to simplify SETCC's.
03600     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03601     if (Simp.getNode()) return Simp;
03602     break;
03603   }
03604   case ISD::SELECT:
03605     if (N1C) {
03606      if (N1C->getZExtValue())
03607        return N2;             // select true, X, Y -> X
03608      return N3;             // select false, X, Y -> Y
03609     }
03610 
03611     if (N2 == N3) return N2;   // select C, X, X -> X
03612     break;
03613   case ISD::VECTOR_SHUFFLE:
03614     llvm_unreachable("should use getVectorShuffle constructor!");
03615   case ISD::INSERT_SUBVECTOR: {
03616     SDValue Index = N3;
03617     if (VT.isSimple() && N1.getValueType().isSimple()
03618         && N2.getValueType().isSimple()) {
03619       assert(VT.isVector() && N1.getValueType().isVector() &&
03620              N2.getValueType().isVector() &&
03621              "Insert subvector VTs must be a vectors");
03622       assert(VT == N1.getValueType() &&
03623              "Dest and insert subvector source types must match!");
03624       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03625              "Insert subvector must be from smaller vector to larger vector!");
03626       if (isa<ConstantSDNode>(Index.getNode())) {
03627         assert((N2.getValueType().getVectorNumElements() +
03628                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03629                 <= VT.getVectorNumElements())
03630                && "Insert subvector overflow!");
03631       }
03632 
03633       // Trivial insertion.
03634       if (VT.getSimpleVT() == N2.getSimpleValueType())
03635         return N2;
03636     }
03637     break;
03638   }
03639   case ISD::BITCAST:
03640     // Fold bit_convert nodes from a type to themselves.
03641     if (N1.getValueType() == VT)
03642       return N1;
03643     break;
03644   }
03645 
03646   // Memoize node if it doesn't produce a flag.
03647   SDNode *N;
03648   SDVTList VTs = getVTList(VT);
03649   if (VT != MVT::Glue) {
03650     SDValue Ops[] = { N1, N2, N3 };
03651     FoldingSetNodeID ID;
03652     AddNodeIDNode(ID, Opcode, VTs, Ops);
03653     void *IP = nullptr;
03654     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03655       return SDValue(E, 0);
03656 
03657     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03658                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03659     CSEMap.InsertNode(N, IP);
03660   } else {
03661     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03662                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03663   }
03664 
03665   InsertNode(N);
03666   return SDValue(N, 0);
03667 }
03668 
03669 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03670                               SDValue N1, SDValue N2, SDValue N3,
03671                               SDValue N4) {
03672   SDValue Ops[] = { N1, N2, N3, N4 };
03673   return getNode(Opcode, DL, VT, Ops);
03674 }
03675 
03676 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03677                               SDValue N1, SDValue N2, SDValue N3,
03678                               SDValue N4, SDValue N5) {
03679   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03680   return getNode(Opcode, DL, VT, Ops);
03681 }
03682 
03683 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03684 /// the incoming stack arguments to be loaded from the stack.
03685 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03686   SmallVector<SDValue, 8> ArgChains;
03687 
03688   // Include the original chain at the beginning of the list. When this is
03689   // used by target LowerCall hooks, this helps legalize find the
03690   // CALLSEQ_BEGIN node.
03691   ArgChains.push_back(Chain);
03692 
03693   // Add a chain value for each stack argument.
03694   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03695        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03696     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03697       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03698         if (FI->getIndex() < 0)
03699           ArgChains.push_back(SDValue(L, 1));
03700 
03701   // Build a tokenfactor for all the chains.
03702   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03703 }
03704 
03705 /// getMemsetValue - Vectorized representation of the memset value
03706 /// operand.
03707 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03708                               SDLoc dl) {
03709   assert(Value.getOpcode() != ISD::UNDEF);
03710 
03711   unsigned NumBits = VT.getScalarType().getSizeInBits();
03712   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03713     assert(C->getAPIntValue().getBitWidth() == 8);
03714     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03715     if (VT.isInteger())
03716       return DAG.getConstant(Val, VT);
03717     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03718   }
03719 
03720   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03721   if (NumBits > 8) {
03722     // Use a multiplication with 0x010101... to extend the input to the
03723     // required length.
03724     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03725     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03726   }
03727 
03728   return Value;
03729 }
03730 
03731 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03732 /// used when a memcpy is turned into a memset when the source is a constant
03733 /// string ptr.
03734 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03735                                   const TargetLowering &TLI, StringRef Str) {
03736   // Handle vector with all elements zero.
03737   if (Str.empty()) {
03738     if (VT.isInteger())
03739       return DAG.getConstant(0, VT);
03740     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03741       return DAG.getConstantFP(0.0, VT);
03742     else if (VT.isVector()) {
03743       unsigned NumElts = VT.getVectorNumElements();
03744       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03745       return DAG.getNode(ISD::BITCAST, dl, VT,
03746                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03747                                                              EltVT, NumElts)));
03748     } else
03749       llvm_unreachable("Expected type!");
03750   }
03751 
03752   assert(!VT.isVector() && "Can't handle vector type here!");
03753   unsigned NumVTBits = VT.getSizeInBits();
03754   unsigned NumVTBytes = NumVTBits / 8;
03755   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03756 
03757   APInt Val(NumVTBits, 0);
03758   if (TLI.isLittleEndian()) {
03759     for (unsigned i = 0; i != NumBytes; ++i)
03760       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03761   } else {
03762     for (unsigned i = 0; i != NumBytes; ++i)
03763       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03764   }
03765 
03766   // If the "cost" of materializing the integer immediate is less than the cost
03767   // of a load, then it is cost effective to turn the load into the immediate.
03768   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03769   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03770     return DAG.getConstant(Val, VT);
03771   return SDValue(nullptr, 0);
03772 }
03773 
03774 /// getMemBasePlusOffset - Returns base and offset node for the
03775 ///
03776 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03777                                       SelectionDAG &DAG) {
03778   EVT VT = Base.getValueType();
03779   return DAG.getNode(ISD::ADD, dl,
03780                      VT, Base, DAG.getConstant(Offset, VT));
03781 }
03782 
03783 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03784 ///
03785 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03786   unsigned SrcDelta = 0;
03787   GlobalAddressSDNode *G = nullptr;
03788   if (Src.getOpcode() == ISD::GlobalAddress)
03789     G = cast<GlobalAddressSDNode>(Src);
03790   else if (Src.getOpcode() == ISD::ADD &&
03791            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03792            Src.getOperand(1).getOpcode() == ISD::Constant) {
03793     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03794     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03795   }
03796   if (!G)
03797     return false;
03798 
03799   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03800 }
03801 
03802 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03803 /// to replace the memset / memcpy. Return true if the number of memory ops
03804 /// is below the threshold. It returns the types of the sequence of
03805 /// memory ops to perform memset / memcpy by reference.
03806 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03807                                      unsigned Limit, uint64_t Size,
03808                                      unsigned DstAlign, unsigned SrcAlign,
03809                                      bool IsMemset,
03810                                      bool ZeroMemset,
03811                                      bool MemcpyStrSrc,
03812                                      bool AllowOverlap,
03813                                      SelectionDAG &DAG,
03814                                      const TargetLowering &TLI) {
03815   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03816          "Expecting memcpy / memset source to meet alignment requirement!");
03817   // If 'SrcAlign' is zero, that means the memory operation does not need to
03818   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03819   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03820   // is the specified alignment of the memory operation. If it is zero, that
03821   // means it's possible to change the alignment of the destination.
03822   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03823   // not need to be loaded.
03824   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03825                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03826                                    DAG.getMachineFunction());
03827 
03828   if (VT == MVT::Other) {
03829     unsigned AS = 0;
03830     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03831         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03832       VT = TLI.getPointerTy();
03833     } else {
03834       switch (DstAlign & 7) {
03835       case 0:  VT = MVT::i64; break;
03836       case 4:  VT = MVT::i32; break;
03837       case 2:  VT = MVT::i16; break;
03838       default: VT = MVT::i8;  break;
03839       }
03840     }
03841 
03842     MVT LVT = MVT::i64;
03843     while (!TLI.isTypeLegal(LVT))
03844       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03845     assert(LVT.isInteger());
03846 
03847     if (VT.bitsGT(LVT))
03848       VT = LVT;
03849   }
03850 
03851   unsigned NumMemOps = 0;
03852   while (Size != 0) {
03853     unsigned VTSize = VT.getSizeInBits() / 8;
03854     while (VTSize > Size) {
03855       // For now, only use non-vector load / store's for the left-over pieces.
03856       EVT NewVT = VT;
03857       unsigned NewVTSize;
03858 
03859       bool Found = false;
03860       if (VT.isVector() || VT.isFloatingPoint()) {
03861         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03862         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03863             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03864           Found = true;
03865         else if (NewVT == MVT::i64 &&
03866                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03867                  TLI.isSafeMemOpType(MVT::f64)) {
03868           // i64 is usually not legal on 32-bit targets, but f64 may be.
03869           NewVT = MVT::f64;
03870           Found = true;
03871         }
03872       }
03873 
03874       if (!Found) {
03875         do {
03876           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03877           if (NewVT == MVT::i8)
03878             break;
03879         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03880       }
03881       NewVTSize = NewVT.getSizeInBits() / 8;
03882 
03883       // If the new VT cannot cover all of the remaining bits, then consider
03884       // issuing a (or a pair of) unaligned and overlapping load / store.
03885       // FIXME: Only does this for 64-bit or more since we don't have proper
03886       // cost model for unaligned load / store.
03887       bool Fast;
03888       unsigned AS = 0;
03889       if (NumMemOps && AllowOverlap &&
03890           VTSize >= 8 && NewVTSize < Size &&
03891           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03892         VTSize = Size;
03893       else {
03894         VT = NewVT;
03895         VTSize = NewVTSize;
03896       }
03897     }
03898 
03899     if (++NumMemOps > Limit)
03900       return false;
03901 
03902     MemOps.push_back(VT);
03903     Size -= VTSize;
03904   }
03905 
03906   return true;
03907 }
03908 
03909 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03910                                        SDValue Chain, SDValue Dst,
03911                                        SDValue Src, uint64_t Size,
03912                                        unsigned Align, bool isVol,
03913                                        bool AlwaysInline,
03914                                        MachinePointerInfo DstPtrInfo,
03915                                        MachinePointerInfo SrcPtrInfo) {
03916   // Turn a memcpy of undef to nop.
03917   if (Src.getOpcode() == ISD::UNDEF)
03918     return Chain;
03919 
03920   // Expand memcpy to a series of load and store ops if the size operand falls
03921   // below a certain threshold.
03922   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03923   // rather than maybe a humongous number of loads and stores.
03924   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03925   std::vector<EVT> MemOps;
03926   bool DstAlignCanChange = false;
03927   MachineFunction &MF = DAG.getMachineFunction();
03928   MachineFrameInfo *MFI = MF.getFrameInfo();
03929   bool OptSize =
03930     MF.getFunction()->getAttributes().
03931       hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
03932   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03933   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
03934     DstAlignCanChange = true;
03935   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
03936   if (Align > SrcAlign)
03937     SrcAlign = Align;
03938   StringRef Str;
03939   bool CopyFromStr = isMemSrcFromString(Src, Str);
03940   bool isZeroStr = CopyFromStr && Str.empty();
03941   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
03942 
03943   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
03944                                 (DstAlignCanChange ? 0 : Align),
03945                                 (isZeroStr ? 0 : SrcAlign),
03946                                 false, false, CopyFromStr, true, DAG, TLI))
03947     return SDValue();
03948 
03949   if (DstAlignCanChange) {
03950     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
03951     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
03952 
03953     // Don't promote to an alignment that would require dynamic stack
03954     // realignment.
03955     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
03956     if (!TRI->needsStackRealignment(MF))
03957        while (NewAlign > Align &&
03958              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
03959           NewAlign /= 2;
03960 
03961     if (NewAlign > Align) {
03962       // Give the stack frame object a larger alignment if needed.
03963       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
03964         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
03965       Align = NewAlign;
03966     }
03967   }
03968 
03969   SmallVector<SDValue, 8> OutChains;
03970   unsigned NumMemOps = MemOps.size();
03971   uint64_t SrcOff = 0, DstOff = 0;
03972   for (unsigned i = 0; i != NumMemOps; ++i) {
03973     EVT VT = MemOps[i];
03974     unsigned VTSize = VT.getSizeInBits() / 8;
03975     SDValue Value, Store;
03976 
03977     if (VTSize > Size) {
03978       // Issuing an unaligned load / store pair  that overlaps with the previous
03979       // pair. Adjust the offset accordingly.
03980       assert(i == NumMemOps-1 && i != 0);
03981       SrcOff -= VTSize - Size;
03982       DstOff -= VTSize - Size;
03983     }
03984 
03985     if (CopyFromStr &&
03986         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
03987       // It's unlikely a store of a vector immediate can be done in a single
03988       // instruction. It would require a load from a constantpool first.
03989       // We only handle zero vectors here.
03990       // FIXME: Handle other cases where store of vector immediate is done in
03991       // a single instruction.
03992       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
03993       if (Value.getNode())
03994         Store = DAG.getStore(Chain, dl, Value,
03995                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03996                              DstPtrInfo.getWithOffset(DstOff), isVol,
03997                              false, Align);
03998     }
03999 
04000     if (!Store.getNode()) {
04001       // The type might not be legal for the target.  This should only happen
04002       // if the type is smaller than a legal type, as on PPC, so the right
04003       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
04004       // to Load/Store if NVT==VT.
04005       // FIXME does the case above also need this?
04006       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
04007       assert(NVT.bitsGE(VT));
04008       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
04009                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04010                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
04011                              false, MinAlign(SrcAlign, SrcOff));
04012       Store = DAG.getTruncStore(Chain, dl, Value,
04013                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04014                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
04015                                 false, Align);
04016     }
04017     OutChains.push_back(Store);
04018     SrcOff += VTSize;
04019     DstOff += VTSize;
04020     Size -= VTSize;
04021   }
04022 
04023   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04024 }
04025 
04026 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04027                                         SDValue Chain, SDValue Dst,
04028                                         SDValue Src, uint64_t Size,
04029                                         unsigned Align,  bool isVol,
04030                                         bool AlwaysInline,
04031                                         MachinePointerInfo DstPtrInfo,
04032                                         MachinePointerInfo SrcPtrInfo) {
04033   // Turn a memmove of undef to nop.
04034   if (Src.getOpcode() == ISD::UNDEF)
04035     return Chain;
04036 
04037   // Expand memmove to a series of load and store ops if the size operand falls
04038   // below a certain threshold.
04039   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04040   std::vector<EVT> MemOps;
04041   bool DstAlignCanChange = false;
04042   MachineFunction &MF = DAG.getMachineFunction();
04043   MachineFrameInfo *MFI = MF.getFrameInfo();
04044   bool OptSize = MF.getFunction()->getAttributes().
04045     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04046   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04047   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04048     DstAlignCanChange = true;
04049   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04050   if (Align > SrcAlign)
04051     SrcAlign = Align;
04052   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04053 
04054   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04055                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04056                                 false, false, false, false, DAG, TLI))
04057     return SDValue();
04058 
04059   if (DstAlignCanChange) {
04060     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04061     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04062     if (NewAlign > Align) {
04063       // Give the stack frame object a larger alignment if needed.
04064       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04065         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04066       Align = NewAlign;
04067     }
04068   }
04069 
04070   uint64_t SrcOff = 0, DstOff = 0;
04071   SmallVector<SDValue, 8> LoadValues;
04072   SmallVector<SDValue, 8> LoadChains;
04073   SmallVector<SDValue, 8> OutChains;
04074   unsigned NumMemOps = MemOps.size();
04075   for (unsigned i = 0; i < NumMemOps; i++) {
04076     EVT VT = MemOps[i];
04077     unsigned VTSize = VT.getSizeInBits() / 8;
04078     SDValue Value;
04079 
04080     Value = DAG.getLoad(VT, dl, Chain,
04081                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04082                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04083                         false, false, SrcAlign);
04084     LoadValues.push_back(Value);
04085     LoadChains.push_back(Value.getValue(1));
04086     SrcOff += VTSize;
04087   }
04088   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04089   OutChains.clear();
04090   for (unsigned i = 0; i < NumMemOps; i++) {
04091     EVT VT = MemOps[i];
04092     unsigned VTSize = VT.getSizeInBits() / 8;
04093     SDValue Store;
04094 
04095     Store = DAG.getStore(Chain, dl, LoadValues[i],
04096                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04097                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04098     OutChains.push_back(Store);
04099     DstOff += VTSize;
04100   }
04101 
04102   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04103 }
04104 
04105 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04106 /// operations.
04107 ///
04108 /// \param DAG Selection DAG where lowered code is placed.
04109 /// \param dl Link to corresponding IR location.
04110 /// \param Chain Control flow dependency.
04111 /// \param Dst Pointer to destination memory location.
04112 /// \param Src Value of byte to write into the memory.
04113 /// \param Size Number of bytes to write.
04114 /// \param Align Alignment of the destination in bytes.
04115 /// \param isVol True if destination is volatile.
04116 /// \param DstPtrInfo IR information on the memory pointer.
04117 /// \returns New head in the control flow, if lowering was successful, empty
04118 /// SDValue otherwise.
04119 ///
04120 /// The function tries to replace 'llvm.memset' intrinsic with several store
04121 /// operations and value calculation code. This is usually profitable for small
04122 /// memory size.
04123 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04124                                SDValue Chain, SDValue Dst,
04125                                SDValue Src, uint64_t Size,
04126                                unsigned Align, bool isVol,
04127                                MachinePointerInfo DstPtrInfo) {
04128   // Turn a memset of undef to nop.
04129   if (Src.getOpcode() == ISD::UNDEF)
04130     return Chain;
04131 
04132   // Expand memset to a series of load/store ops if the size operand
04133   // falls below a certain threshold.
04134   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04135   std::vector<EVT> MemOps;
04136   bool DstAlignCanChange = false;
04137   MachineFunction &MF = DAG.getMachineFunction();
04138   MachineFrameInfo *MFI = MF.getFrameInfo();
04139   bool OptSize = MF.getFunction()->getAttributes().
04140     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04141   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04142   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04143     DstAlignCanChange = true;
04144   bool IsZeroVal =
04145     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04146   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04147                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04148                                 true, IsZeroVal, false, true, DAG, TLI))
04149     return SDValue();
04150 
04151   if (DstAlignCanChange) {
04152     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04153     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04154     if (NewAlign > Align) {
04155       // Give the stack frame object a larger alignment if needed.
04156       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04157         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04158       Align = NewAlign;
04159     }
04160   }
04161 
04162   SmallVector<SDValue, 8> OutChains;
04163   uint64_t DstOff = 0;
04164   unsigned NumMemOps = MemOps.size();
04165 
04166   // Find the largest store and generate the bit pattern for it.
04167   EVT LargestVT = MemOps[0];
04168   for (unsigned i = 1; i < NumMemOps; i++)
04169     if (MemOps[i].bitsGT(LargestVT))
04170       LargestVT = MemOps[i];
04171   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04172 
04173   for (unsigned i = 0; i < NumMemOps; i++) {
04174     EVT VT = MemOps[i];
04175     unsigned VTSize = VT.getSizeInBits() / 8;
04176     if (VTSize > Size) {
04177       // Issuing an unaligned load / store pair  that overlaps with the previous
04178       // pair. Adjust the offset accordingly.
04179       assert(i == NumMemOps-1 && i != 0);
04180       DstOff -= VTSize - Size;
04181     }
04182 
04183     // If this store is smaller than the largest store see whether we can get
04184     // the smaller value for free with a truncate.
04185     SDValue Value = MemSetValue;
04186     if (VT.bitsLT(LargestVT)) {
04187       if (!LargestVT.isVector() && !VT.isVector() &&
04188           TLI.isTruncateFree(LargestVT, VT))
04189         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04190       else
04191         Value = getMemsetValue(Src, VT, DAG, dl);
04192     }
04193     assert(Value.getValueType() == VT && "Value with wrong type.");
04194     SDValue Store = DAG.getStore(Chain, dl, Value,
04195                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04196                                  DstPtrInfo.getWithOffset(DstOff),
04197                                  isVol, false, Align);
04198     OutChains.push_back(Store);
04199     DstOff += VT.getSizeInBits() / 8;
04200     Size -= VTSize;
04201   }
04202 
04203   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04204 }
04205 
04206 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04207                                 SDValue Src, SDValue Size,
04208                                 unsigned Align, bool isVol, bool AlwaysInline,
04209                                 MachinePointerInfo DstPtrInfo,
04210                                 MachinePointerInfo SrcPtrInfo) {
04211   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04212 
04213   // Check to see if we should lower the memcpy to loads and stores first.
04214   // For cases within the target-specified limits, this is the best choice.
04215   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04216   if (ConstantSize) {
04217     // Memcpy with size zero? Just return the original chain.
04218     if (ConstantSize->isNullValue())
04219       return Chain;
04220 
04221     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04222                                              ConstantSize->getZExtValue(),Align,
04223                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04224     if (Result.getNode())
04225       return Result;
04226   }
04227 
04228   // Then check to see if we should lower the memcpy with target-specific
04229   // code. If the target chooses to do this, this is the next best.
04230   SDValue Result =
04231       TSI->EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
04232                                    isVol, AlwaysInline, DstPtrInfo, SrcPtrInfo);
04233   if (Result.getNode())
04234     return Result;
04235 
04236   // If we really need inline code and the target declined to provide it,
04237   // use a (potentially long) sequence of loads and stores.
04238   if (AlwaysInline) {
04239     assert(ConstantSize && "AlwaysInline requires a constant size!");
04240     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04241                                    ConstantSize->getZExtValue(), Align, isVol,
04242                                    true, DstPtrInfo, SrcPtrInfo);
04243   }
04244 
04245   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04246   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04247   // respect volatile, so they may do things like read or write memory
04248   // beyond the given memory regions. But fixing this isn't easy, and most
04249   // people don't care.
04250 
04251   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04252 
04253   // Emit a library call.
04254   TargetLowering::ArgListTy Args;
04255   TargetLowering::ArgListEntry Entry;
04256   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04257   Entry.Node = Dst; Args.push_back(Entry);
04258   Entry.Node = Src; Args.push_back(Entry);
04259   Entry.Node = Size; Args.push_back(Entry);
04260   // FIXME: pass in SDLoc
04261   TargetLowering::CallLoweringInfo CLI(*this);
04262   CLI.setDebugLoc(dl).setChain(Chain)
04263     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04264                Type::getVoidTy(*getContext()),
04265                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04266                                  TLI->getPointerTy()), std::move(Args), 0)
04267     .setDiscardResult();
04268   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04269 
04270   return CallResult.second;
04271 }
04272 
04273 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04274                                  SDValue Src, SDValue Size,
04275                                  unsigned Align, bool isVol,
04276                                  MachinePointerInfo DstPtrInfo,
04277                                  MachinePointerInfo SrcPtrInfo) {
04278   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04279 
04280   // Check to see if we should lower the memmove to loads and stores first.
04281   // For cases within the target-specified limits, this is the best choice.
04282   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04283   if (ConstantSize) {
04284     // Memmove with size zero? Just return the original chain.
04285     if (ConstantSize->isNullValue())
04286       return Chain;
04287 
04288     SDValue Result =
04289       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04290                                ConstantSize->getZExtValue(), Align, isVol,
04291                                false, DstPtrInfo, SrcPtrInfo);
04292     if (Result.getNode())
04293       return Result;
04294   }
04295 
04296   // Then check to see if we should lower the memmove with target-specific
04297   // code. If the target chooses to do this, this is the next best.
04298   SDValue Result = TSI->EmitTargetCodeForMemmove(
04299       *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04300   if (Result.getNode())
04301     return Result;
04302 
04303   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04304   // not be safe.  See memcpy above for more details.
04305 
04306   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04307 
04308   // Emit a library call.
04309   TargetLowering::ArgListTy Args;
04310   TargetLowering::ArgListEntry Entry;
04311   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04312   Entry.Node = Dst; Args.push_back(Entry);
04313   Entry.Node = Src; Args.push_back(Entry);
04314   Entry.Node = Size; Args.push_back(Entry);
04315   // FIXME:  pass in SDLoc
04316   TargetLowering::CallLoweringInfo CLI(*this);
04317   CLI.setDebugLoc(dl).setChain(Chain)
04318     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04319                Type::getVoidTy(*getContext()),
04320                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04321                                  TLI->getPointerTy()), std::move(Args), 0)
04322     .setDiscardResult();
04323   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04324 
04325   return CallResult.second;
04326 }
04327 
04328 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04329                                 SDValue Src, SDValue Size,
04330                                 unsigned Align, bool isVol,
04331                                 MachinePointerInfo DstPtrInfo) {
04332   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04333 
04334   // Check to see if we should lower the memset to stores first.
04335   // For cases within the target-specified limits, this is the best choice.
04336   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04337   if (ConstantSize) {
04338     // Memset with size zero? Just return the original chain.
04339     if (ConstantSize->isNullValue())
04340       return Chain;
04341 
04342     SDValue Result =
04343       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04344                       Align, isVol, DstPtrInfo);
04345 
04346     if (Result.getNode())
04347       return Result;
04348   }
04349 
04350   // Then check to see if we should lower the memset with target-specific
04351   // code. If the target chooses to do this, this is the next best.
04352   SDValue Result = TSI->EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src,
04353                                                 Size, Align, isVol, DstPtrInfo);
04354   if (Result.getNode())
04355     return Result;
04356 
04357   // Emit a library call.
04358   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04359   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04360   TargetLowering::ArgListTy Args;
04361   TargetLowering::ArgListEntry Entry;
04362   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04363   Args.push_back(Entry);
04364   Entry.Node = Src;
04365   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
04366   Args.push_back(Entry);
04367   Entry.Node = Size;
04368   Entry.Ty = IntPtrTy;
04369   Args.push_back(Entry);
04370 
04371   // FIXME: pass in SDLoc
04372   TargetLowering::CallLoweringInfo CLI(*this);
04373   CLI.setDebugLoc(dl).setChain(Chain)
04374     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04375                Type::getVoidTy(*getContext()),
04376                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04377                                  TLI->getPointerTy()), std::move(Args), 0)
04378     .setDiscardResult();
04379 
04380   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04381   return CallResult.second;
04382 }
04383 
04384 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04385                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04386                                 MachineMemOperand *MMO,
04387                                 AtomicOrdering SuccessOrdering,
04388                                 AtomicOrdering FailureOrdering,
04389                                 SynchronizationScope SynchScope) {
04390   FoldingSetNodeID ID;
04391   ID.AddInteger(MemVT.getRawBits());
04392   AddNodeIDNode(ID, Opcode, VTList, Ops);
04393   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04394   void* IP = nullptr;
04395   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04396     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04397     return SDValue(E, 0);
04398   }
04399 
04400   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04401   // SDNode doesn't have access to it.  This memory will be "leaked" when
04402   // the node is deallocated, but recovered when the allocator is released.
04403   // If the number of operands is less than 5 we use AtomicSDNode's internal
04404   // storage.
04405   unsigned NumOps = Ops.size();
04406   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04407                              : nullptr;
04408 
04409   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04410                                                dl.getDebugLoc(), VTList, MemVT,
04411                                                Ops.data(), DynOps, NumOps, MMO,
04412                                                SuccessOrdering, FailureOrdering,
04413                                                SynchScope);
04414   CSEMap.InsertNode(N, IP);
04415   InsertNode(N);
04416   return SDValue(N, 0);
04417 }
04418 
04419 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04420                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04421                                 MachineMemOperand *MMO,
04422                                 AtomicOrdering Ordering,
04423                                 SynchronizationScope SynchScope) {
04424   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04425                    Ordering, SynchScope);
04426 }
04427 
04428 SDValue SelectionDAG::getAtomicCmpSwap(
04429     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04430     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04431     unsigned Alignment, AtomicOrdering SuccessOrdering,
04432     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04433   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04434          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04435   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04436 
04437   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04438     Alignment = getEVTAlignment(MemVT);
04439 
04440   MachineFunction &MF = getMachineFunction();
04441 
04442   // FIXME: Volatile isn't really correct; we should keep track of atomic
04443   // orderings in the memoperand.
04444   unsigned Flags = MachineMemOperand::MOVolatile;
04445   Flags |= MachineMemOperand::MOLoad;
04446   Flags |= MachineMemOperand::MOStore;
04447 
04448   MachineMemOperand *MMO =
04449     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04450 
04451   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04452                           SuccessOrdering, FailureOrdering, SynchScope);
04453 }
04454 
04455 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04456                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04457                                        SDValue Cmp, SDValue Swp,
04458                                        MachineMemOperand *MMO,
04459                                        AtomicOrdering SuccessOrdering,
04460                                        AtomicOrdering FailureOrdering,
04461                                        SynchronizationScope SynchScope) {
04462   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04463          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04464   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04465 
04466   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04467   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04468                    SuccessOrdering, FailureOrdering, SynchScope);
04469 }
04470 
04471 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04472                                 SDValue Chain,
04473                                 SDValue Ptr, SDValue Val,
04474                                 const Value* PtrVal,
04475                                 unsigned Alignment,
04476                                 AtomicOrdering Ordering,
04477                                 SynchronizationScope SynchScope) {
04478   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04479     Alignment = getEVTAlignment(MemVT);
04480 
04481   MachineFunction &MF = getMachineFunction();
04482   // An atomic store does not load. An atomic load does not store.
04483   // (An atomicrmw obviously both loads and stores.)
04484   // For now, atomics are considered to be volatile always, and they are
04485   // chained as such.
04486   // FIXME: Volatile isn't really correct; we should keep track of atomic
04487   // orderings in the memoperand.
04488   unsigned Flags = MachineMemOperand::MOVolatile;
04489   if (Opcode != ISD::ATOMIC_STORE)
04490     Flags |= MachineMemOperand::MOLoad;
04491   if (Opcode != ISD::ATOMIC_LOAD)
04492     Flags |= MachineMemOperand::MOStore;
04493 
04494   MachineMemOperand *MMO =
04495     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04496                             MemVT.getStoreSize(), Alignment);
04497 
04498   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04499                    Ordering, SynchScope);
04500 }
04501 
04502 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04503                                 SDValue Chain,
04504                                 SDValue Ptr, SDValue Val,
04505                                 MachineMemOperand *MMO,
04506                                 AtomicOrdering Ordering,
04507                                 SynchronizationScope SynchScope) {
04508   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04509           Opcode == ISD::ATOMIC_LOAD_SUB ||
04510           Opcode == ISD::ATOMIC_LOAD_AND ||
04511           Opcode == ISD::ATOMIC_LOAD_OR ||
04512           Opcode == ISD::ATOMIC_LOAD_XOR ||
04513           Opcode == ISD::ATOMIC_LOAD_NAND ||
04514           Opcode == ISD::ATOMIC_LOAD_MIN ||
04515           Opcode == ISD::ATOMIC_LOAD_MAX ||
04516           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04517           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04518           Opcode == ISD::ATOMIC_SWAP ||
04519           Opcode == ISD::ATOMIC_STORE) &&
04520          "Invalid Atomic Op");
04521 
04522   EVT VT = Val.getValueType();
04523 
04524   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04525                                                getVTList(VT, MVT::Other);
04526   SDValue Ops[] = {Chain, Ptr, Val};
04527   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04528 }
04529 
04530 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04531                                 EVT VT, SDValue Chain,
04532                                 SDValue Ptr,
04533                                 MachineMemOperand *MMO,
04534                                 AtomicOrdering Ordering,
04535                                 SynchronizationScope SynchScope) {
04536   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04537 
04538   SDVTList VTs = getVTList(VT, MVT::Other);
04539   SDValue Ops[] = {Chain, Ptr};
04540   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04541 }
04542 
04543 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04544 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04545   if (Ops.size() == 1)
04546     return Ops[0];
04547 
04548   SmallVector<EVT, 4> VTs;
04549   VTs.reserve(Ops.size());
04550   for (unsigned i = 0; i < Ops.size(); ++i)
04551     VTs.push_back(Ops[i].getValueType());
04552   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04553 }
04554 
04555 SDValue
04556 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04557                                   ArrayRef<SDValue> Ops,
04558                                   EVT MemVT, MachinePointerInfo PtrInfo,
04559                                   unsigned Align, bool Vol,
04560                                   bool ReadMem, bool WriteMem, unsigned Size) {
04561   if (Align == 0)  // Ensure that codegen never sees alignment 0
04562     Align = getEVTAlignment(MemVT);
04563 
04564   MachineFunction &MF = getMachineFunction();
04565   unsigned Flags = 0;
04566   if (WriteMem)
04567     Flags |= MachineMemOperand::MOStore;
04568   if (ReadMem)
04569     Flags |= MachineMemOperand::MOLoad;
04570   if (Vol)
04571     Flags |= MachineMemOperand::MOVolatile;
04572   if (!Size)
04573     Size = MemVT.getStoreSize();
04574   MachineMemOperand *MMO =
04575     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04576 
04577   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04578 }
04579 
04580 SDValue
04581 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04582                                   ArrayRef<SDValue> Ops, EVT MemVT,
04583                                   MachineMemOperand *MMO) {
04584   assert((Opcode == ISD::INTRINSIC_VOID ||
04585           Opcode == ISD::INTRINSIC_W_CHAIN ||
04586           Opcode == ISD::PREFETCH ||
04587           Opcode == ISD::LIFETIME_START ||
04588           Opcode == ISD::LIFETIME_END ||
04589           (Opcode <= INT_MAX &&
04590            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04591          "Opcode is not a memory-accessing opcode!");
04592 
04593   // Memoize the node unless it returns a flag.
04594   MemIntrinsicSDNode *N;
04595   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04596     FoldingSetNodeID ID;
04597     AddNodeIDNode(ID, Opcode, VTList, Ops);
04598     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04599     void *IP = nullptr;
04600     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04601       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04602       return SDValue(E, 0);
04603     }
04604 
04605     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04606                                                dl.getDebugLoc(), VTList, Ops,
04607                                                MemVT, MMO);
04608     CSEMap.InsertNode(N, IP);
04609   } else {
04610     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04611                                                dl.getDebugLoc(), VTList, Ops,
04612                                                MemVT, MMO);
04613   }
04614   InsertNode(N);
04615   return SDValue(N, 0);
04616 }
04617 
04618 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04619 /// MachinePointerInfo record from it.  This is particularly useful because the
04620 /// code generator has many cases where it doesn't bother passing in a
04621 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04622 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04623   // If this is FI+Offset, we can model it.
04624   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04625     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04626 
04627   // If this is (FI+Offset1)+Offset2, we can model it.
04628   if (Ptr.getOpcode() != ISD::ADD ||
04629       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04630       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04631     return MachinePointerInfo();
04632 
04633   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04634   return MachinePointerInfo::getFixedStack(FI, Offset+
04635                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04636 }
04637 
04638 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04639 /// MachinePointerInfo record from it.  This is particularly useful because the
04640 /// code generator has many cases where it doesn't bother passing in a
04641 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04642 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04643   // If the 'Offset' value isn't a constant, we can't handle this.
04644   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04645     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04646   if (OffsetOp.getOpcode() == ISD::UNDEF)
04647     return InferPointerInfo(Ptr);
04648   return MachinePointerInfo();
04649 }
04650 
04651 
04652 SDValue
04653 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04654                       EVT VT, SDLoc dl, SDValue Chain,
04655                       SDValue Ptr, SDValue Offset,
04656                       MachinePointerInfo PtrInfo, EVT MemVT,
04657                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04658                       unsigned Alignment, const AAMDNodes &AAInfo,
04659                       const MDNode *Ranges) {
04660   assert(Chain.getValueType() == MVT::Other &&
04661         "Invalid chain type");
04662   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04663     Alignment = getEVTAlignment(VT);
04664 
04665   unsigned Flags = MachineMemOperand::MOLoad;
04666   if (isVolatile)
04667     Flags |= MachineMemOperand::MOVolatile;
04668   if (isNonTemporal)
04669     Flags |= MachineMemOperand::MONonTemporal;
04670   if (isInvariant)
04671     Flags |= MachineMemOperand::MOInvariant;
04672 
04673   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04674   // clients.
04675   if (PtrInfo.V.isNull())
04676     PtrInfo = InferPointerInfo(Ptr, Offset);
04677 
04678   MachineFunction &MF = getMachineFunction();
04679   MachineMemOperand *MMO =
04680     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04681                             AAInfo, Ranges);
04682   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04683 }
04684 
04685 SDValue
04686 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04687                       EVT VT, SDLoc dl, SDValue Chain,
04688                       SDValue Ptr, SDValue Offset, EVT MemVT,
04689                       MachineMemOperand *MMO) {
04690   if (VT == MemVT) {
04691     ExtType = ISD::NON_EXTLOAD;
04692   } else if (ExtType == ISD::NON_EXTLOAD) {
04693     assert(VT == MemVT && "Non-extending load from different memory type!");
04694   } else {
04695     // Extending load.
04696     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04697            "Should only be an extending load, not truncating!");
04698     assert(VT.isInteger() == MemVT.isInteger() &&
04699            "Cannot convert from FP to Int or Int -> FP!");
04700     assert(VT.isVector() == MemVT.isVector() &&
04701            "Cannot use trunc store to convert to or from a vector!");
04702     assert((!VT.isVector() ||
04703             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04704            "Cannot use trunc store to change the number of vector elements!");
04705   }
04706 
04707   bool Indexed = AM != ISD::UNINDEXED;
04708   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04709          "Unindexed load with an offset!");
04710 
04711   SDVTList VTs = Indexed ?
04712     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04713   SDValue Ops[] = { Chain, Ptr, Offset };
04714   FoldingSetNodeID ID;
04715   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04716   ID.AddInteger(MemVT.getRawBits());
04717   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04718                                      MMO->isNonTemporal(),
04719                                      MMO->isInvariant()));
04720   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04721   void *IP = nullptr;
04722   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04723     cast<LoadSDNode>(E)->refineAlignment(MMO);
04724     return SDValue(E, 0);
04725   }
04726   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04727                                              dl.getDebugLoc(), VTs, AM, ExtType,
04728                                              MemVT, MMO);
04729   CSEMap.InsertNode(N, IP);
04730   InsertNode(N);
04731   return SDValue(N, 0);
04732 }
04733 
04734 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04735                               SDValue Chain, SDValue Ptr,
04736                               MachinePointerInfo PtrInfo,
04737                               bool isVolatile, bool isNonTemporal,
04738                               bool isInvariant, unsigned Alignment,
04739                               const AAMDNodes &AAInfo,
04740                               const MDNode *Ranges) {
04741   SDValue Undef = getUNDEF(Ptr.getValueType());
04742   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04743                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04744                  AAInfo, Ranges);
04745 }
04746 
04747 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04748                               SDValue Chain, SDValue Ptr,
04749                               MachineMemOperand *MMO) {
04750   SDValue Undef = getUNDEF(Ptr.getValueType());
04751   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04752                  VT, MMO);
04753 }
04754 
04755 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04756                                  SDValue Chain, SDValue Ptr,
04757                                  MachinePointerInfo PtrInfo, EVT MemVT,
04758                                  bool isVolatile, bool isNonTemporal,
04759                                  bool isInvariant, unsigned Alignment,
04760                                  const AAMDNodes &AAInfo) {
04761   SDValue Undef = getUNDEF(Ptr.getValueType());
04762   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04763                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04764                  Alignment, AAInfo);
04765 }
04766 
04767 
04768 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04769                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04770                                  MachineMemOperand *MMO) {
04771   SDValue Undef = getUNDEF(Ptr.getValueType());
04772   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04773                  MemVT, MMO);
04774 }
04775 
04776 SDValue
04777 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04778                              SDValue Offset, ISD::MemIndexedMode AM) {
04779   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04780   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04781          "Load is already a indexed load!");
04782   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04783                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04784                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04785                  false, LD->getAlignment());
04786 }
04787 
04788 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04789                                SDValue Ptr, MachinePointerInfo PtrInfo,
04790                                bool isVolatile, bool isNonTemporal,
04791                                unsigned Alignment, const AAMDNodes &AAInfo) {
04792   assert(Chain.getValueType() == MVT::Other &&
04793         "Invalid chain type");
04794   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04795     Alignment = getEVTAlignment(Val.getValueType());
04796 
04797   unsigned Flags = MachineMemOperand::MOStore;
04798   if (isVolatile)
04799     Flags |= MachineMemOperand::MOVolatile;
04800   if (isNonTemporal)
04801     Flags |= MachineMemOperand::MONonTemporal;
04802 
04803   if (PtrInfo.V.isNull())
04804     PtrInfo = InferPointerInfo(Ptr);
04805 
04806   MachineFunction &MF = getMachineFunction();
04807   MachineMemOperand *MMO =
04808     MF.getMachineMemOperand(PtrInfo, Flags,
04809                             Val.getValueType().getStoreSize(), Alignment,
04810                             AAInfo);
04811 
04812   return getStore(Chain, dl, Val, Ptr, MMO);
04813 }
04814 
04815 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04816                                SDValue Ptr, MachineMemOperand *MMO) {
04817   assert(Chain.getValueType() == MVT::Other &&
04818         "Invalid chain type");
04819   EVT VT = Val.getValueType();
04820   SDVTList VTs = getVTList(MVT::Other);
04821   SDValue Undef = getUNDEF(Ptr.getValueType());
04822   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04823   FoldingSetNodeID ID;
04824   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04825   ID.AddInteger(VT.getRawBits());
04826   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04827                                      MMO->isNonTemporal(), MMO->isInvariant()));
04828   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04829   void *IP = nullptr;
04830   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04831     cast<StoreSDNode>(E)->refineAlignment(MMO);
04832     return SDValue(E, 0);
04833   }
04834   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04835                                               dl.getDebugLoc(), VTs,
04836                                               ISD::UNINDEXED, false, VT, MMO);
04837   CSEMap.InsertNode(N, IP);
04838   InsertNode(N);
04839   return SDValue(N, 0);
04840 }
04841 
04842 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04843                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04844                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04845                                     unsigned Alignment,
04846                                     const AAMDNodes &AAInfo) {
04847   assert(Chain.getValueType() == MVT::Other &&
04848         "Invalid chain type");
04849   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04850     Alignment = getEVTAlignment(SVT);
04851 
04852   unsigned Flags = MachineMemOperand::MOStore;
04853   if (isVolatile)
04854     Flags |= MachineMemOperand::MOVolatile;
04855   if (isNonTemporal)
04856     Flags |= MachineMemOperand::MONonTemporal;
04857 
04858   if (PtrInfo.V.isNull())
04859     PtrInfo = InferPointerInfo(Ptr);
04860 
04861   MachineFunction &MF = getMachineFunction();
04862   MachineMemOperand *MMO =
04863     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04864                             AAInfo);
04865 
04866   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04867 }
04868 
04869 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04870                                     SDValue Ptr, EVT SVT,
04871                                     MachineMemOperand *MMO) {
04872   EVT VT = Val.getValueType();
04873 
04874   assert(Chain.getValueType() == MVT::Other &&
04875         "Invalid chain type");
04876   if (VT == SVT)
04877     return getStore(Chain, dl, Val, Ptr, MMO);
04878 
04879   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04880          "Should only be a truncating store, not extending!");
04881   assert(VT.isInteger() == SVT.isInteger() &&
04882          "Can't do FP-INT conversion!");
04883   assert(VT.isVector() == SVT.isVector() &&
04884          "Cannot use trunc store to convert to or from a vector!");
04885   assert((!VT.isVector() ||
04886           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04887          "Cannot use trunc store to change the number of vector elements!");
04888 
04889   SDVTList VTs = getVTList(MVT::Other);
04890   SDValue Undef = getUNDEF(Ptr.getValueType());
04891   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04892   FoldingSetNodeID ID;
04893   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04894   ID.AddInteger(SVT.getRawBits());
04895   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04896                                      MMO->isNonTemporal(), MMO->isInvariant()));
04897   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04898   void *IP = nullptr;
04899   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04900     cast<StoreSDNode>(E)->refineAlignment(MMO);
04901     return SDValue(E, 0);
04902   }
04903   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04904                                               dl.getDebugLoc(), VTs,
04905                                               ISD::UNINDEXED, true, SVT, MMO);
04906   CSEMap.InsertNode(N, IP);
04907   InsertNode(N);
04908   return SDValue(N, 0);
04909 }
04910 
04911 SDValue
04912 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04913                               SDValue Offset, ISD::MemIndexedMode AM) {
04914   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04915   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04916          "Store is already a indexed store!");
04917   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04918   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04919   FoldingSetNodeID ID;
04920   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04921   ID.AddInteger(ST->getMemoryVT().getRawBits());
04922   ID.AddInteger(ST->getRawSubclassData());
04923   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04924   void *IP = nullptr;
04925   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04926     return SDValue(E, 0);
04927 
04928   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04929                                               dl.getDebugLoc(), VTs, AM,
04930                                               ST->isTruncatingStore(),
04931                                               ST->getMemoryVT(),
04932                                               ST->getMemOperand());
04933   CSEMap.InsertNode(N, IP);
04934   InsertNode(N);
04935   return SDValue(N, 0);
04936 }
04937 
04938 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
04939                                SDValue Chain, SDValue Ptr,
04940                                SDValue SV,
04941                                unsigned Align) {
04942   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
04943   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
04944 }
04945 
04946 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04947                               ArrayRef<SDUse> Ops) {
04948   switch (Ops.size()) {
04949   case 0: return getNode(Opcode, DL, VT);
04950   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
04951   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04952   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04953   default: break;
04954   }
04955 
04956   // Copy from an SDUse array into an SDValue array for use with
04957   // the regular getNode logic.
04958   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
04959   return getNode(Opcode, DL, VT, NewOps);
04960 }
04961 
04962 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04963                               ArrayRef<SDValue> Ops) {
04964   unsigned NumOps = Ops.size();
04965   switch (NumOps) {
04966   case 0: return getNode(Opcode, DL, VT);
04967   case 1: return getNode(Opcode, DL, VT, Ops[0]);
04968   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04969   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04970   default: break;
04971   }
04972 
04973   switch (Opcode) {
04974   default: break;
04975   case ISD::SELECT_CC: {
04976     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
04977     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
04978            "LHS and RHS of condition must have same type!");
04979     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04980            "True and False arms of SelectCC must have same type!");
04981     assert(Ops[2].getValueType() == VT &&
04982            "select_cc node must be of same type as true and false value!");
04983     break;
04984   }
04985   case ISD::BR_CC: {
04986     assert(NumOps == 5 && "BR_CC takes 5 operands!");
04987     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04988            "LHS/RHS of comparison should match types!");
04989     break;
04990   }
04991   }
04992 
04993   // Memoize nodes.
04994   SDNode *N;
04995   SDVTList VTs = getVTList(VT);
04996 
04997   if (VT != MVT::Glue) {
04998     FoldingSetNodeID ID;
04999     AddNodeIDNode(ID, Opcode, VTs, Ops);
05000     void *IP = nullptr;
05001 
05002     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05003       return SDValue(E, 0);
05004 
05005     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05006                                    VTs, Ops);
05007     CSEMap.InsertNode(N, IP);
05008   } else {
05009     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05010                                    VTs, Ops);
05011   }
05012 
05013   InsertNode(N);
05014   return SDValue(N, 0);
05015 }
05016 
05017 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05018                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05019   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05020 }
05021 
05022 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05023                               ArrayRef<SDValue> Ops) {
05024   if (VTList.NumVTs == 1)
05025     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05026 
05027 #if 0
05028   switch (Opcode) {
05029   // FIXME: figure out how to safely handle things like
05030   // int foo(int x) { return 1 << (x & 255); }
05031   // int bar() { return foo(256); }
05032   case ISD::SRA_PARTS:
05033   case ISD::SRL_PARTS:
05034   case ISD::SHL_PARTS:
05035     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05036         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05037       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05038     else if (N3.getOpcode() == ISD::AND)
05039       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05040         // If the and is only masking out bits that cannot effect the shift,
05041         // eliminate the and.
05042         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05043         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05044           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05045       }
05046     break;
05047   }
05048 #endif
05049 
05050   // Memoize the node unless it returns a flag.
05051   SDNode *N;
05052   unsigned NumOps = Ops.size();
05053   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05054     FoldingSetNodeID ID;
05055     AddNodeIDNode(ID, Opcode, VTList, Ops);
05056     void *IP = nullptr;
05057     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05058       return SDValue(E, 0);
05059 
05060     if (NumOps == 1) {
05061       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05062                                           DL.getDebugLoc(), VTList, Ops[0]);
05063     } else if (NumOps == 2) {
05064       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05065                                            DL.getDebugLoc(), VTList, Ops[0],
05066                                            Ops[1]);
05067     } else if (NumOps == 3) {
05068       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05069                                             DL.getDebugLoc(), VTList, Ops[0],
05070                                             Ops[1], Ops[2]);
05071     } else {
05072       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05073                                      VTList, Ops);
05074     }
05075     CSEMap.InsertNode(N, IP);
05076   } else {
05077     if (NumOps == 1) {
05078       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05079                                           DL.getDebugLoc(), VTList, Ops[0]);
05080     } else if (NumOps == 2) {
05081       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05082                                            DL.getDebugLoc(), VTList, Ops[0],
05083                                            Ops[1]);
05084     } else if (NumOps == 3) {
05085       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05086                                             DL.getDebugLoc(), VTList, Ops[0],
05087                                             Ops[1], Ops[2]);
05088     } else {
05089       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05090                                      VTList, Ops);
05091     }
05092   }
05093   InsertNode(N);
05094   return SDValue(N, 0);
05095 }
05096 
05097 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05098   return getNode(Opcode, DL, VTList, None);
05099 }
05100 
05101 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05102                               SDValue N1) {
05103   SDValue Ops[] = { N1 };
05104   return getNode(Opcode, DL, VTList, Ops);
05105 }
05106 
05107 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05108                               SDValue N1, SDValue N2) {
05109   SDValue Ops[] = { N1, N2 };
05110   return getNode(Opcode, DL, VTList, Ops);
05111 }
05112 
05113 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05114                               SDValue N1, SDValue N2, SDValue N3) {
05115   SDValue Ops[] = { N1, N2, N3 };
05116   return getNode(Opcode, DL, VTList, Ops);
05117 }
05118 
05119 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05120                               SDValue N1, SDValue N2, SDValue N3,
05121                               SDValue N4) {
05122   SDValue Ops[] = { N1, N2, N3, N4 };
05123   return getNode(Opcode, DL, VTList, Ops);
05124 }
05125 
05126 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05127                               SDValue N1, SDValue N2, SDValue N3,
05128                               SDValue N4, SDValue N5) {
05129   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05130   return getNode(Opcode, DL, VTList, Ops);
05131 }
05132 
05133 SDVTList SelectionDAG::getVTList(EVT VT) {
05134   return makeVTList(SDNode::getValueTypeList(VT), 1);
05135 }
05136 
05137 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05138   FoldingSetNodeID ID;
05139   ID.AddInteger(2U);
05140   ID.AddInteger(VT1.getRawBits());
05141   ID.AddInteger(VT2.getRawBits());
05142 
05143   void *IP = nullptr;
05144   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05145   if (!Result) {
05146     EVT *Array = Allocator.Allocate<EVT>(2);
05147     Array[0] = VT1;
05148     Array[1] = VT2;
05149     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05150     VTListMap.InsertNode(Result, IP);
05151   }
05152   return Result->getSDVTList();
05153 }
05154 
05155 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05156   FoldingSetNodeID ID;
05157   ID.AddInteger(3U);
05158   ID.AddInteger(VT1.getRawBits());
05159   ID.AddInteger(VT2.getRawBits());
05160   ID.AddInteger(VT3.getRawBits());
05161 
05162   void *IP = nullptr;
05163   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05164   if (!Result) {
05165     EVT *Array = Allocator.Allocate<EVT>(3);
05166     Array[0] = VT1;
05167     Array[1] = VT2;
05168     Array[2] = VT3;
05169     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05170     VTListMap.InsertNode(Result, IP);
05171   }
05172   return Result->getSDVTList();
05173 }
05174 
05175 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05176   FoldingSetNodeID ID;
05177   ID.AddInteger(4U);
05178   ID.AddInteger(VT1.getRawBits());
05179   ID.AddInteger(VT2.getRawBits());
05180   ID.AddInteger(VT3.getRawBits());
05181   ID.AddInteger(VT4.getRawBits());
05182 
05183   void *IP = nullptr;
05184   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05185   if (!Result) {
05186     EVT *Array = Allocator.Allocate<EVT>(4);
05187     Array[0] = VT1;
05188     Array[1] = VT2;
05189     Array[2] = VT3;
05190     Array[3] = VT4;
05191     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05192     VTListMap.InsertNode(Result, IP);
05193   }
05194   return Result->getSDVTList();
05195 }
05196 
05197 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05198   unsigned NumVTs = VTs.size();
05199   FoldingSetNodeID ID;
05200   ID.AddInteger(NumVTs);
05201   for (unsigned index = 0; index < NumVTs; index++) {
05202     ID.AddInteger(VTs[index].getRawBits());
05203   }
05204 
05205   void *IP = nullptr;
05206   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05207   if (!Result) {
05208     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05209     std::copy(VTs.begin(), VTs.end(), Array);
05210     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05211     VTListMap.InsertNode(Result, IP);
05212   }
05213   return Result->getSDVTList();
05214 }
05215 
05216 
05217 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05218 /// specified operands.  If the resultant node already exists in the DAG,
05219 /// this does not modify the specified node, instead it returns the node that
05220 /// already exists.  If the resultant node does not exist in the DAG, the
05221 /// input node is returned.  As a degenerate case, if you specify the same
05222 /// input operands as the node already has, the input node is returned.
05223 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05224   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05225 
05226   // Check to see if there is no change.
05227   if (Op == N->getOperand(0)) return N;
05228 
05229   // See if the modified node already exists.
05230   void *InsertPos = nullptr;
05231   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05232     return Existing;
05233 
05234   // Nope it doesn't.  Remove the node from its current place in the maps.
05235   if (InsertPos)
05236     if (!RemoveNodeFromCSEMaps(N))
05237       InsertPos = nullptr;
05238 
05239   // Now we update the operands.
05240   N->OperandList[0].set(Op);
05241 
05242   // If this gets put into a CSE map, add it.
05243   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05244   return N;
05245 }
05246 
05247 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05248   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05249 
05250   // Check to see if there is no change.
05251   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05252     return N;   // No operands changed, just return the input node.
05253 
05254   // See if the modified node already exists.
05255   void *InsertPos = nullptr;
05256   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05257     return Existing;
05258 
05259   // Nope it doesn't.  Remove the node from its current place in the maps.
05260   if (InsertPos)
05261     if (!RemoveNodeFromCSEMaps(N))
05262       InsertPos = nullptr;
05263 
05264   // Now we update the operands.
05265   if (N->OperandList[0] != Op1)
05266     N->OperandList[0].set(Op1);
05267   if (N->OperandList[1] != Op2)
05268     N->OperandList[1].set(Op2);
05269 
05270   // If this gets put into a CSE map, add it.
05271   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05272   return N;
05273 }
05274 
05275 SDNode *SelectionDAG::
05276 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05277   SDValue Ops[] = { Op1, Op2, Op3 };
05278   return UpdateNodeOperands(N, Ops);
05279 }
05280 
05281 SDNode *SelectionDAG::
05282 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05283                    SDValue Op3, SDValue Op4) {
05284   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05285   return UpdateNodeOperands(N, Ops);
05286 }
05287 
05288 SDNode *SelectionDAG::
05289 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05290                    SDValue Op3, SDValue Op4, SDValue Op5) {
05291   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05292   return UpdateNodeOperands(N, Ops);
05293 }
05294 
05295 SDNode *SelectionDAG::
05296 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05297   unsigned NumOps = Ops.size();
05298   assert(N->getNumOperands() == NumOps &&
05299          "Update with wrong number of operands");
05300 
05301   // Check to see if there is no change.
05302   bool AnyChange = false;
05303   for (unsigned i = 0; i != NumOps; ++i) {
05304     if (Ops[i] != N->getOperand(i)) {
05305       AnyChange = true;
05306       break;
05307     }
05308   }
05309 
05310   // No operands changed, just return the input node.
05311   if (!AnyChange) return N;
05312 
05313   // See if the modified node already exists.
05314   void *InsertPos = nullptr;
05315   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05316     return Existing;
05317 
05318   // Nope it doesn't.  Remove the node from its current place in the maps.
05319   if (InsertPos)
05320     if (!RemoveNodeFromCSEMaps(N))
05321       InsertPos = nullptr;
05322 
05323   // Now we update the operands.
05324   for (unsigned i = 0; i != NumOps; ++i)
05325     if (N->OperandList[i] != Ops[i])
05326       N->OperandList[i].set(Ops[i]);
05327 
05328   // If this gets put into a CSE map, add it.
05329   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05330   return N;
05331 }
05332 
05333 /// DropOperands - Release the operands and set this node to have
05334 /// zero operands.
05335 void SDNode::DropOperands() {
05336   // Unlike the code in MorphNodeTo that does this, we don't need to
05337   // watch for dead nodes here.
05338   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05339     SDUse &Use = *I++;
05340     Use.set(SDValue());
05341   }
05342 }
05343 
05344 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05345 /// machine opcode.
05346 ///
05347 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05348                                    EVT VT) {
05349   SDVTList VTs = getVTList(VT);
05350   return SelectNodeTo(N, MachineOpc, VTs, None);
05351 }
05352 
05353 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05354                                    EVT VT, SDValue Op1) {
05355   SDVTList VTs = getVTList(VT);
05356   SDValue Ops[] = { Op1 };
05357   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05358 }
05359 
05360 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05361                                    EVT VT, SDValue Op1,
05362                                    SDValue Op2) {
05363   SDVTList VTs = getVTList(VT);
05364   SDValue Ops[] = { Op1, Op2 };
05365   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05366 }
05367 
05368 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05369                                    EVT VT, SDValue Op1,
05370                                    SDValue Op2, SDValue Op3) {
05371   SDVTList VTs = getVTList(VT);
05372   SDValue Ops[] = { Op1, Op2, Op3 };
05373   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05374 }
05375 
05376 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05377                                    EVT VT, ArrayRef<SDValue> Ops) {
05378   SDVTList VTs = getVTList(VT);
05379   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05380 }
05381 
05382 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05383                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05384   SDVTList VTs = getVTList(VT1, VT2);
05385   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05386 }
05387 
05388 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05389                                    EVT VT1, EVT VT2) {
05390   SDVTList VTs = getVTList(VT1, VT2);
05391   return SelectNodeTo(N, MachineOpc, VTs, None);
05392 }
05393 
05394 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05395                                    EVT VT1, EVT VT2, EVT VT3,
05396                                    ArrayRef<SDValue> Ops) {
05397   SDVTList VTs = getVTList(VT1, VT2, VT3);
05398   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05399 }
05400 
05401 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05402                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05403                                    ArrayRef<SDValue> Ops) {
05404   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05405   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05406 }
05407 
05408 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05409                                    EVT VT1, EVT VT2,
05410                                    SDValue Op1) {
05411   SDVTList VTs = getVTList(VT1, VT2);
05412   SDValue Ops[] = { Op1 };
05413   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05414 }
05415 
05416 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05417                                    EVT VT1, EVT VT2,
05418                                    SDValue Op1, SDValue Op2) {
05419   SDVTList VTs = getVTList(VT1, VT2);
05420   SDValue Ops[] = { Op1, Op2 };
05421   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05422 }
05423 
05424 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05425                                    EVT VT1, EVT VT2,
05426                                    SDValue Op1, SDValue Op2,
05427                                    SDValue Op3) {
05428   SDVTList VTs = getVTList(VT1, VT2);
05429   SDValue Ops[] = { Op1, Op2, Op3 };
05430   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05431 }
05432 
05433 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05434                                    EVT VT1, EVT VT2, EVT VT3,
05435                                    SDValue Op1, SDValue Op2,
05436                                    SDValue Op3) {
05437   SDVTList VTs = getVTList(VT1, VT2, VT3);
05438   SDValue Ops[] = { Op1, Op2, Op3 };
05439   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05440 }
05441 
05442 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05443                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05444   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05445   // Reset the NodeID to -1.
05446   N->setNodeId(-1);
05447   return N;
05448 }
05449 
05450 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05451 /// the line number information on the merged node since it is not possible to
05452 /// preserve the information that operation is associated with multiple lines.
05453 /// This will make the debugger working better at -O0, were there is a higher
05454 /// probability having other instructions associated with that line.
05455 ///
05456 /// For IROrder, we keep the smaller of the two
05457 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05458   DebugLoc NLoc = N->getDebugLoc();
05459   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05460     (OLoc.getDebugLoc() != NLoc)) {
05461     N->setDebugLoc(DebugLoc());
05462   }
05463   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05464   N->setIROrder(Order);
05465   return N;
05466 }
05467 
05468 /// MorphNodeTo - This *mutates* the specified node to have the specified
05469 /// return type, opcode, and operands.
05470 ///
05471 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05472 /// node of the specified opcode and operands, it returns that node instead of
05473 /// the current one.  Note that the SDLoc need not be the same.
05474 ///
05475 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05476 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05477 /// node, and because it doesn't require CSE recalculation for any of
05478 /// the node's users.
05479 ///
05480 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05481 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05482 /// the legalizer which maintain worklists that would need to be updated when
05483 /// deleting things.
05484 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05485                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05486   unsigned NumOps = Ops.size();
05487   // If an identical node already exists, use it.
05488   void *IP = nullptr;
05489   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05490     FoldingSetNodeID ID;
05491     AddNodeIDNode(ID, Opc, VTs, Ops);
05492     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05493       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05494   }
05495 
05496   if (!RemoveNodeFromCSEMaps(N))
05497     IP = nullptr;
05498 
05499   // Start the morphing.
05500   N->NodeType = Opc;
05501   N->ValueList = VTs.VTs;
05502   N->NumValues = VTs.NumVTs;
05503 
05504   // Clear the operands list, updating used nodes to remove this from their
05505   // use list.  Keep track of any operands that become dead as a result.
05506   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05507   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05508     SDUse &Use = *I++;
05509     SDNode *Used = Use.getNode();
05510     Use.set(SDValue());
05511     if (Used->use_empty())
05512       DeadNodeSet.insert(Used);
05513   }
05514 
05515   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05516     // Initialize the memory references information.
05517     MN->setMemRefs(nullptr, nullptr);
05518     // If NumOps is larger than the # of operands we can have in a
05519     // MachineSDNode, reallocate the operand list.
05520     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05521       if (MN->OperandsNeedDelete)
05522         delete[] MN->OperandList;
05523       if (NumOps > array_lengthof(MN->LocalOperands))
05524         // We're creating a final node that will live unmorphed for the
05525         // remainder of the current SelectionDAG iteration, so we can allocate
05526         // the operands directly out of a pool with no recycling metadata.
05527         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05528                          Ops.data(), NumOps);
05529       else
05530         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05531       MN->OperandsNeedDelete = false;
05532     } else
05533       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05534   } else {
05535     // If NumOps is larger than the # of operands we currently have, reallocate
05536     // the operand list.
05537     if (NumOps > N->NumOperands) {
05538       if (N->OperandsNeedDelete)
05539         delete[] N->OperandList;
05540       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05541       N->OperandsNeedDelete = true;
05542     } else
05543       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05544   }
05545 
05546   // Delete any nodes that are still dead after adding the uses for the
05547   // new operands.
05548   if (!DeadNodeSet.empty()) {
05549     SmallVector<SDNode *, 16> DeadNodes;
05550     for (SDNode *N : DeadNodeSet)
05551       if (N->use_empty())
05552         DeadNodes.push_back(N);
05553     RemoveDeadNodes(DeadNodes);
05554   }
05555 
05556   if (IP)
05557     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05558   return N;
05559 }
05560 
05561 
05562 /// getMachineNode - These are used for target selectors to create a new node
05563 /// with specified return type(s), MachineInstr opcode, and operands.
05564 ///
05565 /// Note that getMachineNode returns the resultant node.  If there is already a
05566 /// node of the specified opcode and operands, it returns that node instead of
05567 /// the current one.
05568 MachineSDNode *
05569 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05570   SDVTList VTs = getVTList(VT);
05571   return getMachineNode(Opcode, dl, VTs, None);
05572 }
05573 
05574 MachineSDNode *
05575 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05576   SDVTList VTs = getVTList(VT);
05577   SDValue Ops[] = { Op1 };
05578   return getMachineNode(Opcode, dl, VTs, Ops);
05579 }
05580 
05581 MachineSDNode *
05582 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05583                              SDValue Op1, SDValue Op2) {
05584   SDVTList VTs = getVTList(VT);
05585   SDValue Ops[] = { Op1, Op2 };
05586   return getMachineNode(Opcode, dl, VTs, Ops);
05587 }
05588 
05589 MachineSDNode *
05590 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05591                              SDValue Op1, SDValue Op2, SDValue Op3) {
05592   SDVTList VTs = getVTList(VT);
05593   SDValue Ops[] = { Op1, Op2, Op3 };
05594   return getMachineNode(Opcode, dl, VTs, Ops);
05595 }
05596 
05597 MachineSDNode *
05598 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05599                              ArrayRef<SDValue> Ops) {
05600   SDVTList VTs = getVTList(VT);
05601   return getMachineNode(Opcode, dl, VTs, Ops);
05602 }
05603 
05604 MachineSDNode *
05605 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05606   SDVTList VTs = getVTList(VT1, VT2);
05607   return getMachineNode(Opcode, dl, VTs, None);
05608 }
05609 
05610 MachineSDNode *
05611 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05612                              EVT VT1, EVT VT2, SDValue Op1) {
05613   SDVTList VTs = getVTList(VT1, VT2);
05614   SDValue Ops[] = { Op1 };
05615   return getMachineNode(Opcode, dl, VTs, Ops);
05616 }
05617 
05618 MachineSDNode *
05619 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05620                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05621   SDVTList VTs = getVTList(VT1, VT2);
05622   SDValue Ops[] = { Op1, Op2 };
05623   return getMachineNode(Opcode, dl, VTs, Ops);
05624 }
05625 
05626 MachineSDNode *
05627 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05628                              EVT VT1, EVT VT2, SDValue Op1,
05629                              SDValue Op2, SDValue Op3) {
05630   SDVTList VTs = getVTList(VT1, VT2);
05631   SDValue Ops[] = { Op1, Op2, Op3 };
05632   return getMachineNode(Opcode, dl, VTs, Ops);
05633 }
05634 
05635 MachineSDNode *
05636 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05637                              EVT VT1, EVT VT2,
05638                              ArrayRef<SDValue> Ops) {
05639   SDVTList VTs = getVTList(VT1, VT2);
05640   return getMachineNode(Opcode, dl, VTs, Ops);
05641 }
05642 
05643 MachineSDNode *
05644 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05645                              EVT VT1, EVT VT2, EVT VT3,
05646                              SDValue Op1, SDValue Op2) {
05647   SDVTList VTs = getVTList(VT1, VT2, VT3);
05648   SDValue Ops[] = { Op1, Op2 };
05649   return getMachineNode(Opcode, dl, VTs, Ops);
05650 }
05651 
05652 MachineSDNode *
05653 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05654                              EVT VT1, EVT VT2, EVT VT3,
05655                              SDValue Op1, SDValue Op2, SDValue Op3) {
05656   SDVTList VTs = getVTList(VT1, VT2, VT3);
05657   SDValue Ops[] = { Op1, Op2, Op3 };
05658   return getMachineNode(Opcode, dl, VTs, Ops);
05659 }
05660 
05661 MachineSDNode *
05662 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05663                              EVT VT1, EVT VT2, EVT VT3,
05664                              ArrayRef<SDValue> Ops) {
05665   SDVTList VTs = getVTList(VT1, VT2, VT3);
05666   return getMachineNode(Opcode, dl, VTs, Ops);
05667 }
05668 
05669 MachineSDNode *
05670 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05671                              EVT VT2, EVT VT3, EVT VT4,
05672                              ArrayRef<SDValue> Ops) {
05673   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05674   return getMachineNode(Opcode, dl, VTs, Ops);
05675 }
05676 
05677 MachineSDNode *
05678 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05679                              ArrayRef<EVT> ResultTys,
05680                              ArrayRef<SDValue> Ops) {
05681   SDVTList VTs = getVTList(ResultTys);
05682   return getMachineNode(Opcode, dl, VTs, Ops);
05683 }
05684 
05685 MachineSDNode *
05686 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05687                              ArrayRef<SDValue> OpsArray) {
05688   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05689   MachineSDNode *N;
05690   void *IP = nullptr;
05691   const SDValue *Ops = OpsArray.data();
05692   unsigned NumOps = OpsArray.size();
05693 
05694   if (DoCSE) {
05695     FoldingSetNodeID ID;
05696     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05697     IP = nullptr;
05698     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05699       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05700     }
05701   }
05702 
05703   // Allocate a new MachineSDNode.
05704   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05705                                         DL.getDebugLoc(), VTs);
05706 
05707   // Initialize the operands list.
05708   if (NumOps > array_lengthof(N->LocalOperands))
05709     // We're creating a final node that will live unmorphed for the
05710     // remainder of the current SelectionDAG iteration, so we can allocate
05711     // the operands directly out of a pool with no recycling metadata.
05712     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05713                     Ops, NumOps);
05714   else
05715     N->InitOperands(N->LocalOperands, Ops, NumOps);
05716   N->OperandsNeedDelete = false;
05717 
05718   if (DoCSE)
05719     CSEMap.InsertNode(N, IP);
05720 
05721   InsertNode(N);
05722   return N;
05723 }
05724 
05725 /// getTargetExtractSubreg - A convenience function for creating
05726 /// TargetOpcode::EXTRACT_SUBREG nodes.
05727 SDValue
05728 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05729                                      SDValue Operand) {
05730   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05731   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05732                                   VT, Operand, SRIdxVal);
05733   return SDValue(Subreg, 0);
05734 }
05735 
05736 /// getTargetInsertSubreg - A convenience function for creating
05737 /// TargetOpcode::INSERT_SUBREG nodes.
05738 SDValue
05739 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05740                                     SDValue Operand, SDValue Subreg) {
05741   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05742   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05743                                   VT, Operand, Subreg, SRIdxVal);
05744   return SDValue(Result, 0);
05745 }
05746 
05747 /// getNodeIfExists - Get the specified node if it's already available, or
05748 /// else return NULL.
05749 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05750                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05751                                       bool exact) {
05752   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05753     FoldingSetNodeID ID;
05754     AddNodeIDNode(ID, Opcode, VTList, Ops);
05755     if (isBinOpWithFlags(Opcode))
05756       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05757     void *IP = nullptr;
05758     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05759       return E;
05760   }
05761   return nullptr;
05762 }
05763 
05764 /// getDbgValue - Creates a SDDbgValue node.
05765 ///
05766 /// SDNode
05767 SDDbgValue *
05768 SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R,
05769         bool IsIndirect, uint64_t Off,
05770                           DebugLoc DL, unsigned O) {
05771   return new (Allocator) SDDbgValue(MDPtr, N, R, IsIndirect, Off, DL, O);
05772 }
05773 
05774 /// Constant
05775 SDDbgValue *
05776 SelectionDAG::getConstantDbgValue(MDNode *MDPtr, const Value *C,
05777           uint64_t Off,
05778           DebugLoc DL, unsigned O) {
05779   return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O);
05780 }
05781 
05782 /// FrameIndex
05783 SDDbgValue *
05784 SelectionDAG::getFrameIndexDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
05785             DebugLoc DL, unsigned O) {
05786   return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O);
05787 }
05788 
05789 namespace {
05790 
05791 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05792 /// pointed to by a use iterator is deleted, increment the use iterator
05793 /// so that it doesn't dangle.
05794 ///
05795 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05796   SDNode::use_iterator &UI;
05797   SDNode::use_iterator &UE;
05798 
05799   void NodeDeleted(SDNode *N, SDNode *E) override {
05800     // Increment the iterator as needed.
05801     while (UI != UE && N == *UI)
05802       ++UI;
05803   }
05804 
05805 public:
05806   RAUWUpdateListener(SelectionDAG &d,
05807                      SDNode::use_iterator &ui,
05808                      SDNode::use_iterator &ue)
05809     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05810 };
05811 
05812 }
05813 
05814 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05815 /// This can cause recursive merging of nodes in the DAG.
05816 ///
05817 /// This version assumes From has a single result value.
05818 ///
05819 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05820   SDNode *From = FromN.getNode();
05821   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05822          "Cannot replace with this method!");
05823   assert(From != To.getNode() && "Cannot replace uses of with self");
05824 
05825   // Iterate over all the existing uses of From. New uses will be added
05826   // to the beginning of the use list, which we avoid visiting.
05827   // This specifically avoids visiting uses of From that arise while the
05828   // replacement is happening, because any such uses would be the result
05829   // of CSE: If an existing node looks like From after one of its operands
05830   // is replaced by To, we don't want to replace of all its users with To
05831   // too. See PR3018 for more info.
05832   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05833   RAUWUpdateListener Listener(*this, UI, UE);
05834   while (UI != UE) {
05835     SDNode *User = *UI;
05836 
05837     // This node is about to morph, remove its old self from the CSE maps.
05838     RemoveNodeFromCSEMaps(User);
05839 
05840     // A user can appear in a use list multiple times, and when this
05841     // happens the uses are usually next to each other in the list.
05842     // To help reduce the number of CSE recomputations, process all
05843     // the uses of this user that we can find this way.
05844     do {
05845       SDUse &Use = UI.getUse();
05846       ++UI;
05847       Use.set(To);
05848     } while (UI != UE && *UI == User);
05849 
05850     // Now that we have modified User, add it back to the CSE maps.  If it
05851     // already exists there, recursively merge the results together.
05852     AddModifiedNodeToCSEMaps(User);
05853   }
05854 
05855   // If we just RAUW'd the root, take note.
05856   if (FromN == getRoot())
05857     setRoot(To);
05858 }
05859 
05860 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05861 /// This can cause recursive merging of nodes in the DAG.
05862 ///
05863 /// This version assumes that for each value of From, there is a
05864 /// corresponding value in To in the same position with the same type.
05865 ///
05866 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05867 #ifndef NDEBUG
05868   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05869     assert((!From->hasAnyUseOfValue(i) ||
05870             From->getValueType(i) == To->getValueType(i)) &&
05871            "Cannot use this version of ReplaceAllUsesWith!");
05872 #endif
05873 
05874   // Handle the trivial case.
05875   if (From == To)
05876     return;
05877 
05878   // Iterate over just the existing users of From. See the comments in
05879   // the ReplaceAllUsesWith above.
05880   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05881   RAUWUpdateListener Listener(*this, UI, UE);
05882   while (UI != UE) {
05883     SDNode *User = *UI;
05884 
05885     // This node is about to morph, remove its old self from the CSE maps.
05886     RemoveNodeFromCSEMaps(User);
05887 
05888     // A user can appear in a use list multiple times, and when this
05889     // happens the uses are usually next to each other in the list.
05890     // To help reduce the number of CSE recomputations, process all
05891     // the uses of this user that we can find this way.
05892     do {
05893       SDUse &Use = UI.getUse();
05894       ++UI;
05895       Use.setNode(To);
05896     } while (UI != UE && *UI == User);
05897 
05898     // Now that we have modified User, add it back to the CSE maps.  If it
05899     // already exists there, recursively merge the results together.
05900     AddModifiedNodeToCSEMaps(User);
05901   }
05902 
05903   // If we just RAUW'd the root, take note.
05904   if (From == getRoot().getNode())
05905     setRoot(SDValue(To, getRoot().getResNo()));
05906 }
05907 
05908 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05909 /// This can cause recursive merging of nodes in the DAG.
05910 ///
05911 /// This version can replace From with any result values.  To must match the
05912 /// number and types of values returned by From.
05913 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
05914   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
05915     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
05916 
05917   // Iterate over just the existing users of From. See the comments in
05918   // the ReplaceAllUsesWith above.
05919   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05920   RAUWUpdateListener Listener(*this, UI, UE);
05921   while (UI != UE) {
05922     SDNode *User = *UI;
05923 
05924     // This node is about to morph, remove its old self from the CSE maps.
05925     RemoveNodeFromCSEMaps(User);
05926 
05927     // A user can appear in a use list multiple times, and when this
05928     // happens the uses are usually next to each other in the list.
05929     // To help reduce the number of CSE recomputations, process all
05930     // the uses of this user that we can find this way.
05931     do {
05932       SDUse &Use = UI.getUse();
05933       const SDValue &ToOp = To[Use.getResNo()];
05934       ++UI;
05935       Use.set(ToOp);
05936     } while (UI != UE && *UI == User);
05937 
05938     // Now that we have modified User, add it back to the CSE maps.  If it
05939     // already exists there, recursively merge the results together.
05940     AddModifiedNodeToCSEMaps(User);
05941   }
05942 
05943   // If we just RAUW'd the root, take note.
05944   if (From == getRoot().getNode())
05945     setRoot(SDValue(To[getRoot().getResNo()]));
05946 }
05947 
05948 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
05949 /// uses of other values produced by From.getNode() alone.  The Deleted
05950 /// vector is handled the same way as for ReplaceAllUsesWith.
05951 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
05952   // Handle the really simple, really trivial case efficiently.
05953   if (From == To) return;
05954 
05955   // Handle the simple, trivial, case efficiently.
05956   if (From.getNode()->getNumValues() == 1) {
05957     ReplaceAllUsesWith(From, To);
05958     return;
05959   }
05960 
05961   // Iterate over just the existing users of From. See the comments in
05962   // the ReplaceAllUsesWith above.
05963   SDNode::use_iterator UI = From.getNode()->use_begin(),
05964                        UE = From.getNode()->use_end();
05965   RAUWUpdateListener Listener(*this, UI, UE);
05966   while (UI != UE) {
05967     SDNode *User = *UI;
05968     bool UserRemovedFromCSEMaps = false;
05969 
05970     // A user can appear in a use list multiple times, and when this
05971     // happens the uses are usually next to each other in the list.
05972     // To help reduce the number of CSE recomputations, process all
05973     // the uses of this user that we can find this way.
05974     do {
05975       SDUse &Use = UI.getUse();
05976 
05977       // Skip uses of different values from the same node.
05978       if (Use.getResNo() != From.getResNo()) {
05979         ++UI;
05980         continue;
05981       }
05982 
05983       // If this node hasn't been modified yet, it's still in the CSE maps,
05984       // so remove its old self from the CSE maps.
05985       if (!UserRemovedFromCSEMaps) {
05986         RemoveNodeFromCSEMaps(User);
05987         UserRemovedFromCSEMaps = true;
05988       }
05989 
05990       ++UI;
05991       Use.set(To);
05992     } while (UI != UE && *UI == User);
05993 
05994     // We are iterating over all uses of the From node, so if a use
05995     // doesn't use the specific value, no changes are made.
05996     if (!UserRemovedFromCSEMaps)
05997       continue;
05998 
05999     // Now that we have modified User, add it back to the CSE maps.  If it
06000     // already exists there, recursively merge the results together.
06001     AddModifiedNodeToCSEMaps(User);
06002   }
06003 
06004   // If we just RAUW'd the root, take note.
06005   if (From == getRoot())
06006     setRoot(To);
06007 }
06008 
06009 namespace {
06010   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
06011   /// to record information about a use.
06012   struct UseMemo {
06013     SDNode *User;
06014     unsigned Index;
06015     SDUse *Use;
06016   };
06017 
06018   /// operator< - Sort Memos by User.
06019   bool operator<(const UseMemo &L, const UseMemo &R) {
06020     return (intptr_t)L.User < (intptr_t)R.User;
06021   }
06022 }
06023 
06024 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06025 /// uses of other values produced by From.getNode() alone.  The same value
06026 /// may appear in both the From and To list.  The Deleted vector is
06027 /// handled the same way as for ReplaceAllUsesWith.
06028 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06029                                               const SDValue *To,
06030                                               unsigned Num){
06031   // Handle the simple, trivial case efficiently.
06032   if (Num == 1)
06033     return ReplaceAllUsesOfValueWith(*From, *To);
06034 
06035   // Read up all the uses and make records of them. This helps
06036   // processing new uses that are introduced during the
06037   // replacement process.
06038   SmallVector<UseMemo, 4> Uses;
06039   for (unsigned i = 0; i != Num; ++i) {
06040     unsigned FromResNo = From[i].getResNo();
06041     SDNode *FromNode = From[i].getNode();
06042     for (SDNode::use_iterator UI = FromNode->use_begin(),
06043          E = FromNode->use_end(); UI != E; ++UI) {
06044       SDUse &Use = UI.getUse();
06045       if (Use.getResNo() == FromResNo) {
06046         UseMemo Memo = { *UI, i, &Use };
06047         Uses.push_back(Memo);
06048       }
06049     }
06050   }
06051 
06052   // Sort the uses, so that all the uses from a given User are together.
06053   std::sort(Uses.begin(), Uses.end());
06054 
06055   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06056        UseIndex != UseIndexEnd; ) {
06057     // We know that this user uses some value of From.  If it is the right
06058     // value, update it.
06059     SDNode *User = Uses[UseIndex].User;
06060 
06061     // This node is about to morph, remove its old self from the CSE maps.
06062     RemoveNodeFromCSEMaps(User);
06063 
06064     // The Uses array is sorted, so all the uses for a given User
06065     // are next to each other in the list.
06066     // To help reduce the number of CSE recomputations, process all
06067     // the uses of this user that we can find this way.
06068     do {
06069       unsigned i = Uses[UseIndex].Index;
06070       SDUse &Use = *Uses[UseIndex].Use;
06071       ++UseIndex;
06072 
06073       Use.set(To[i]);
06074     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06075 
06076     // Now that we have modified User, add it back to the CSE maps.  If it
06077     // already exists there, recursively merge the results together.
06078     AddModifiedNodeToCSEMaps(User);
06079   }
06080 }
06081 
06082 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06083 /// based on their topological order. It returns the maximum id and a vector
06084 /// of the SDNodes* in assigned order by reference.
06085 unsigned SelectionDAG::AssignTopologicalOrder() {
06086 
06087   unsigned DAGSize = 0;
06088 
06089   // SortedPos tracks the progress of the algorithm. Nodes before it are
06090   // sorted, nodes after it are unsorted. When the algorithm completes
06091   // it is at the end of the list.
06092   allnodes_iterator SortedPos = allnodes_begin();
06093 
06094   // Visit all the nodes. Move nodes with no operands to the front of
06095   // the list immediately. Annotate nodes that do have operands with their
06096   // operand count. Before we do this, the Node Id fields of the nodes
06097   // may contain arbitrary values. After, the Node Id fields for nodes
06098   // before SortedPos will contain the topological sort index, and the
06099   // Node Id fields for nodes At SortedPos and after will contain the
06100   // count of outstanding operands.
06101   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06102     SDNode *N = I++;
06103     checkForCycles(N, this);
06104     unsigned Degree = N->getNumOperands();
06105     if (Degree == 0) {
06106       // A node with no uses, add it to the result array immediately.
06107       N->setNodeId(DAGSize++);
06108       allnodes_iterator Q = N;
06109       if (Q != SortedPos)
06110         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06111       assert(SortedPos != AllNodes.end() && "Overran node list");
06112       ++SortedPos;
06113     } else {
06114       // Temporarily use the Node Id as scratch space for the degree count.
06115       N->setNodeId(Degree);
06116     }
06117   }
06118 
06119   // Visit all the nodes. As we iterate, move nodes into sorted order,
06120   // such that by the time the end is reached all nodes will be sorted.
06121   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06122     SDNode *N = I;
06123     checkForCycles(N, this);
06124     // N is in sorted position, so all its uses have one less operand
06125     // that needs to be sorted.
06126     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06127          UI != UE; ++UI) {
06128       SDNode *P = *UI;
06129       unsigned Degree = P->getNodeId();
06130       assert(Degree != 0 && "Invalid node degree");
06131       --Degree;
06132       if (Degree == 0) {
06133         // All of P's operands are sorted, so P may sorted now.
06134         P->setNodeId(DAGSize++);
06135         if (P != SortedPos)
06136           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06137         assert(SortedPos != AllNodes.end() && "Overran node list");
06138         ++SortedPos;
06139       } else {
06140         // Update P's outstanding operand count.
06141         P->setNodeId(Degree);
06142       }
06143     }
06144     if (I == SortedPos) {
06145 #ifndef NDEBUG
06146       SDNode *S = ++I;
06147       dbgs() << "Overran sorted position:\n";
06148       S->dumprFull(this); dbgs() << "\n";
06149       dbgs() << "Checking if this is due to cycles\n";
06150       checkForCycles(this, true);
06151 #endif
06152       llvm_unreachable(nullptr);
06153     }
06154   }
06155 
06156   assert(SortedPos == AllNodes.end() &&
06157          "Topological sort incomplete!");
06158   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06159          "First node in topological sort is not the entry token!");
06160   assert(AllNodes.front().getNodeId() == 0 &&
06161          "First node in topological sort has non-zero id!");
06162   assert(AllNodes.front().getNumOperands() == 0 &&
06163          "First node in topological sort has operands!");
06164   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06165          "Last node in topologic sort has unexpected id!");
06166   assert(AllNodes.back().use_empty() &&
06167          "Last node in topologic sort has users!");
06168   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06169   return DAGSize;
06170 }
06171 
06172 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06173 /// value is produced by SD.
06174 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06175   DbgInfo->add(DB, SD, isParameter);
06176   if (SD)
06177     SD->setHasDebugValue(true);
06178 }
06179 
06180 /// TransferDbgValues - Transfer SDDbgValues.
06181 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06182   if (From == To || !From.getNode()->getHasDebugValue())
06183     return;
06184   SDNode *FromNode = From.getNode();
06185   SDNode *ToNode = To.getNode();
06186   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
06187   SmallVector<SDDbgValue *, 2> ClonedDVs;
06188   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
06189        I != E; ++I) {
06190     SDDbgValue *Dbg = *I;
06191     if (Dbg->getKind() == SDDbgValue::SDNODE) {
06192       SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(),
06193               Dbg->isIndirect(),
06194                                       Dbg->getOffset(), Dbg->getDebugLoc(),
06195                                       Dbg->getOrder());
06196       ClonedDVs.push_back(Clone);
06197     }
06198   }
06199   for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
06200          E = ClonedDVs.end(); I != E; ++I)
06201     AddDbgValue(*I, ToNode, false);
06202 }
06203 
06204 //===----------------------------------------------------------------------===//
06205 //                              SDNode Class
06206 //===----------------------------------------------------------------------===//
06207 
06208 HandleSDNode::~HandleSDNode() {
06209   DropOperands();
06210 }
06211 
06212 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
06213                                          DebugLoc DL, const GlobalValue *GA,
06214                                          EVT VT, int64_t o, unsigned char TF)
06215   : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
06216   TheGlobal = GA;
06217 }
06218 
06219 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
06220                                          SDValue X, unsigned SrcAS,
06221                                          unsigned DestAS)
06222  : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
06223    SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
06224 
06225 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06226                      EVT memvt, MachineMemOperand *mmo)
06227  : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
06228   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06229                                       MMO->isNonTemporal(), MMO->isInvariant());
06230   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06231   assert(isNonTemporal() == MMO->isNonTemporal() &&
06232          "Non-temporal encoding error!");
06233   // We check here that the size of the memory operand fits within the size of
06234   // the MMO. This is because the MMO might indicate only a possible address
06235   // range instead of specifying the affected memory addresses precisely.
06236   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06237 }
06238 
06239 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06240                      ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
06241    : SDNode(Opc, Order, dl, VTs, Ops),
06242      MemoryVT(memvt), MMO(mmo) {
06243   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06244                                       MMO->isNonTemporal(), MMO->isInvariant());
06245   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06246   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06247 }
06248 
06249 /// Profile - Gather unique data for the node.
06250 ///
06251 void SDNode::Profile(FoldingSetNodeID &ID) const {
06252   AddNodeIDNode(ID, this);
06253 }
06254 
06255 namespace {
06256   struct EVTArray {
06257     std::vector<EVT> VTs;
06258 
06259     EVTArray() {
06260       VTs.reserve(MVT::LAST_VALUETYPE);
06261       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
06262         VTs.push_back(MVT((MVT::SimpleValueType)i));
06263     }
06264   };
06265 }
06266 
06267 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
06268 static ManagedStatic<EVTArray> SimpleVTArray;
06269 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
06270 
06271 /// getValueTypeList - Return a pointer to the specified value type.
06272 ///
06273 const EVT *SDNode::getValueTypeList(EVT VT) {
06274   if (VT.isExtended()) {
06275     sys::SmartScopedLock<true> Lock(*VTMutex);
06276     return &(*EVTs->insert(VT).first);
06277   } else {
06278     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
06279            "Value type out of range!");
06280     return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
06281   }
06282 }
06283 
06284 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
06285 /// indicated value.  This method ignores uses of other values defined by this
06286 /// operation.
06287 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
06288   assert(Value < getNumValues() && "Bad value!");
06289 
06290   // TODO: Only iterate over uses of a given value of the node
06291   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
06292     if (UI.getUse().getResNo() == Value) {
06293       if (NUses == 0)
06294         return false;
06295       --NUses;
06296     }
06297   }
06298 
06299   // Found exactly the right number of uses?
06300   return NUses == 0;
06301 }
06302 
06303 
06304 /// hasAnyUseOfValue - Return true if there are any use of the indicated
06305 /// value. This method ignores uses of other values defined by this operation.
06306 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
06307   assert(Value < getNumValues() && "Bad value!");
06308 
06309   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
06310     if (UI.getUse().getResNo() == Value)
06311