LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - SelectionDAG.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 3426 3569 96.0 %
Date: 2018-02-23 15:42:53 Functions: 271 277 97.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This implements the SelectionDAG class.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/CodeGen/SelectionDAG.h"
      15             : #include "SDNodeDbgValue.h"
      16             : #include "llvm/ADT/APFloat.h"
      17             : #include "llvm/ADT/APInt.h"
      18             : #include "llvm/ADT/APSInt.h"
      19             : #include "llvm/ADT/ArrayRef.h"
      20             : #include "llvm/ADT/BitVector.h"
      21             : #include "llvm/ADT/FoldingSet.h"
      22             : #include "llvm/ADT/None.h"
      23             : #include "llvm/ADT/STLExtras.h"
      24             : #include "llvm/ADT/SmallPtrSet.h"
      25             : #include "llvm/ADT/SmallVector.h"
      26             : #include "llvm/ADT/Triple.h"
      27             : #include "llvm/ADT/Twine.h"
      28             : #include "llvm/Analysis/ValueTracking.h"
      29             : #include "llvm/CodeGen/ISDOpcodes.h"
      30             : #include "llvm/CodeGen/MachineBasicBlock.h"
      31             : #include "llvm/CodeGen/MachineConstantPool.h"
      32             : #include "llvm/CodeGen/MachineFrameInfo.h"
      33             : #include "llvm/CodeGen/MachineFunction.h"
      34             : #include "llvm/CodeGen/MachineMemOperand.h"
      35             : #include "llvm/CodeGen/MachineValueType.h"
      36             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      37             : #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
      38             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      39             : #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
      40             : #include "llvm/CodeGen/TargetLowering.h"
      41             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      42             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      43             : #include "llvm/CodeGen/ValueTypes.h"
      44             : #include "llvm/IR/Constant.h"
      45             : #include "llvm/IR/Constants.h"
      46             : #include "llvm/IR/DataLayout.h"
      47             : #include "llvm/IR/DebugInfoMetadata.h"
      48             : #include "llvm/IR/DebugLoc.h"
      49             : #include "llvm/IR/DerivedTypes.h"
      50             : #include "llvm/IR/Function.h"
      51             : #include "llvm/IR/GlobalValue.h"
      52             : #include "llvm/IR/Metadata.h"
      53             : #include "llvm/IR/Type.h"
      54             : #include "llvm/IR/Value.h"
      55             : #include "llvm/Support/Casting.h"
      56             : #include "llvm/Support/CodeGen.h"
      57             : #include "llvm/Support/Compiler.h"
      58             : #include "llvm/Support/Debug.h"
      59             : #include "llvm/Support/ErrorHandling.h"
      60             : #include "llvm/Support/KnownBits.h"
      61             : #include "llvm/Support/ManagedStatic.h"
      62             : #include "llvm/Support/MathExtras.h"
      63             : #include "llvm/Support/Mutex.h"
      64             : #include "llvm/Support/raw_ostream.h"
      65             : #include "llvm/Target/TargetMachine.h"
      66             : #include "llvm/Target/TargetOptions.h"
      67             : #include <algorithm>
      68             : #include <cassert>
      69             : #include <cstdint>
      70             : #include <cstdlib>
      71             : #include <limits>
      72             : #include <set>
      73             : #include <string>
      74             : #include <utility>
      75             : #include <vector>
      76             : 
      77             : using namespace llvm;
      78             : 
      79             : /// makeVTList - Return an instance of the SDVTList struct initialized with the
      80             : /// specified members.
      81             : static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
      82             :   SDVTList Res = {VTs, NumVTs};
      83             :   return Res;
      84             : }
      85             : 
      86             : // Default null implementations of the callbacks.
      87           0 : void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
      88    13966689 : void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
      89             : 
      90             : #define DEBUG_TYPE "selectiondag"
      91             : 
      92             : static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
      93             :   DEBUG(
      94             :     dbgs() << Msg;
      95             :     V.getNode()->dump(G);
      96             :   );
      97             : }
      98             : 
      99             : //===----------------------------------------------------------------------===//
     100             : //                              ConstantFPSDNode Class
     101             : //===----------------------------------------------------------------------===//
     102             : 
     103             : /// isExactlyValue - We don't rely on operator== working on double values, as
     104             : /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
     105             : /// As such, this method can be used to do an exact bit-for-bit comparison of
     106             : /// two floating point values.
     107           3 : bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
     108           6 :   return getValueAPF().bitwiseIsEqual(V);
     109             : }
     110             : 
     111        1559 : bool ConstantFPSDNode::isValueValidForType(EVT VT,
     112             :                                            const APFloat& Val) {
     113             :   assert(VT.isFloatingPoint() && "Can only convert between FP types");
     114             : 
     115             :   // convert modifies in place, so make a copy.
     116             :   APFloat Val2 = APFloat(Val);
     117             :   bool losesInfo;
     118        1559 :   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
     119             :                       APFloat::rmNearestTiesToEven,
     120             :                       &losesInfo);
     121        3118 :   return !losesInfo;
     122             : }
     123             : 
     124             : //===----------------------------------------------------------------------===//
     125             : //                              ISD Namespace
     126             : //===----------------------------------------------------------------------===//
     127             : 
     128      512290 : bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
     129             :   auto *BV = dyn_cast<BuildVectorSDNode>(N);
     130             :   if (!BV)
     131             :     return false;
     132             : 
     133             :   APInt SplatUndef;
     134             :   unsigned SplatBitSize;
     135             :   bool HasUndefs;
     136      577286 :   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
     137      288643 :   return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
     138      576922 :                              EltSize) &&
     139      288279 :          EltSize == SplatBitSize;
     140             : }
     141             : 
     142             : // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
     143             : // specializations of the more general isConstantSplatVector()?
     144             : 
     145      309292 : bool ISD::isBuildVectorAllOnes(const SDNode *N) {
     146             :   // Look through a bit convert.
     147      809018 :   while (N->getOpcode() == ISD::BITCAST)
     148       63478 :     N = N->getOperand(0).getNode();
     149             : 
     150      309292 :   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
     151             : 
     152      162066 :   unsigned i = 0, e = N->getNumOperands();
     153             : 
     154             :   // Skip over all of the undef values.
     155      492210 :   while (i != e && N->getOperand(i).isUndef())
     156        1520 :     ++i;
     157             : 
     158             :   // Do not accept an all-undef vector.
     159      162066 :   if (i == e) return false;
     160             : 
     161             :   // Do not accept build_vectors that aren't all constants or which have non-~0
     162             :   // elements. We have to be a bit careful here, as the type of the constant
     163             :   // may not be the same as the type of the vector elements due to type
     164             :   // legalization (the elements are promoted to a legal type for the target and
     165             :   // a vector of a type may be legal when the base element type is not).
     166             :   // We only want to check enough bits to cover the vector elements, because
     167             :   // we care if the resultant vector is all ones, not whether the individual
     168             :   // constants are.
     169      324064 :   SDValue NotZero = N->getOperand(i);
     170      486096 :   unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
     171             :   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
     172      302128 :     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
     173             :       return false;
     174             :   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
     175        8877 :     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
     176             :       return false;
     177             :   } else
     178             :     return false;
     179             : 
     180             :   // Okay, we have at least one ~0 value, check to see if the rest match or are
     181             :   // undefs. Even with the above element type twiddling, this should be OK, as
     182             :   // the same type legalization should have applied to all the elements.
     183      470670 :   for (++i; i != e; ++i)
     184      356319 :     if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
     185             :       return false;
     186             :   return true;
     187             : }
     188             : 
     189     1282868 : bool ISD::isBuildVectorAllZeros(const SDNode *N) {
     190             :   // Look through a bit convert.
     191     3420034 :   while (N->getOpcode() == ISD::BITCAST)
     192      284766 :     N = N->getOperand(0).getNode();
     193             : 
     194     1282868 :   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
     195             : 
     196             :   bool IsAllUndef = true;
     197     1717407 :   for (const SDValue &Op : N->op_values()) {
     198     3257966 :     if (Op.isUndef())
     199        8757 :       continue;
     200             :     IsAllUndef = false;
     201             :     // Do not accept build_vectors that aren't all constants or which have non-0
     202             :     // elements. We have to be a bit careful here, as the type of the constant
     203             :     // may not be the same as the type of the vector elements due to type
     204             :     // legalization (the elements are promoted to a legal type for the target
     205             :     // and a vector of a type may be legal when the base element type is not).
     206             :     // We only want to check enough bits to cover the vector elements, because
     207             :     // we care if the resultant vector is all zeros, not whether the individual
     208             :     // constants are.
     209     4860678 :     unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
     210             :     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
     211     3149476 :       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
     212             :         return false;
     213             :     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
     214      101664 :       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
     215             :         return false;
     216             :     } else
     217             :       return false;
     218             :   }
     219             : 
     220             :   // Do not accept an all-undef vector.
     221       88424 :   if (IsAllUndef)
     222             :     return false;
     223       88389 :   return true;
     224             : }
     225             : 
     226     4097747 : bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
     227     4097747 :   if (N->getOpcode() != ISD::BUILD_VECTOR)
     228             :     return false;
     229             : 
     230      439056 :   for (const SDValue &Op : N->op_values()) {
     231      811126 :     if (Op.isUndef())
     232        5661 :       continue;
     233             :     if (!isa<ConstantSDNode>(Op))
     234             :       return false;
     235             :   }
     236             :   return true;
     237             : }
     238             : 
     239     1004290 : bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
     240     1004290 :   if (N->getOpcode() != ISD::BUILD_VECTOR)
     241             :     return false;
     242             : 
     243       24148 :   for (const SDValue &Op : N->op_values()) {
     244       41444 :     if (Op.isUndef())
     245         690 :       continue;
     246             :     if (!isa<ConstantFPSDNode>(Op))
     247             :       return false;
     248             :   }
     249             :   return true;
     250             : }
     251             : 
     252      353584 : bool ISD::allOperandsUndef(const SDNode *N) {
     253             :   // Return false if the node has no operands.
     254             :   // This is "logically inconsistent" with the definition of "all" but
     255             :   // is probably the desired behavior.
     256      707168 :   if (N->getNumOperands() == 0)
     257             :     return false;
     258             : 
     259      357833 :   for (const SDValue &Op : N->op_values())
     260      715474 :     if (!Op.isUndef())
     261             :       return false;
     262             : 
     263             :   return true;
     264             : }
     265             : 
     266      329358 : bool ISD::matchUnaryPredicate(SDValue Op,
     267             :                               std::function<bool(ConstantSDNode *)> Match) {
     268             :   if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
     269      305420 :     return Match(Cst);
     270             : 
     271       23938 :   if (ISD::BUILD_VECTOR != Op.getOpcode())
     272             :     return false;
     273             : 
     274       11240 :   EVT SVT = Op.getValueType().getScalarType();
     275        5958 :   for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
     276             :     auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
     277       16676 :     if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
     278             :       return false;
     279             :   }
     280             :   return true;
     281             : }
     282             : 
     283       39266 : bool ISD::matchBinaryPredicate(
     284             :     SDValue LHS, SDValue RHS,
     285             :     std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match) {
     286       39306 :   if (LHS.getValueType() != RHS.getValueType())
     287             :     return false;
     288             : 
     289             :   if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
     290             :     if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
     291       16012 :       return Match(LHSCst, RHSCst);
     292             : 
     293       22202 :   if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
     294             :       ISD::BUILD_VECTOR != RHS.getOpcode())
     295             :     return false;
     296             : 
     297         193 :   EVT SVT = LHS.getValueType().getScalarType();
     298        2537 :   for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
     299             :     auto *LHSCst = dyn_cast<ConstantSDNode>(LHS.getOperand(i));
     300             :     auto *RHSCst = dyn_cast<ConstantSDNode>(RHS.getOperand(i));
     301        1208 :     if (!LHSCst || !RHSCst)
     302             :       return false;
     303        2416 :     if (LHSCst->getValueType(0) != SVT ||
     304        1208 :         LHSCst->getValueType(0) != RHSCst->getValueType(0))
     305             :       return false;
     306        1208 :     if (!Match(LHSCst, RHSCst))
     307             :       return false;
     308             :   }
     309             :   return true;
     310             : }
     311             : 
     312         315 : ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
     313         315 :   switch (ExtType) {
     314           0 :   case ISD::EXTLOAD:
     315           0 :     return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
     316             :   case ISD::SEXTLOAD:
     317             :     return ISD::SIGN_EXTEND;
     318         188 :   case ISD::ZEXTLOAD:
     319         188 :     return ISD::ZERO_EXTEND;
     320             :   default:
     321             :     break;
     322             :   }
     323             : 
     324           0 :   llvm_unreachable("Invalid LoadExtType");
     325             : }
     326             : 
     327      294710 : ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
     328             :   // To perform this operation, we just need to swap the L and G bits of the
     329             :   // operation.
     330      294710 :   unsigned OldL = (Operation >> 2) & 1;
     331      294710 :   unsigned OldG = (Operation >> 1) & 1;
     332      589420 :   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
     333      294710 :                        (OldL << 1) |       // New G bit
     334      294710 :                        (OldG << 2));       // New L bit.
     335             : }
     336             : 
     337       58814 : ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
     338             :   unsigned Operation = Op;
     339       58814 :   if (isInteger)
     340       57273 :     Operation ^= 7;   // Flip L, G, E bits, but not U.
     341             :   else
     342        1541 :     Operation ^= 15;  // Flip all of the condition bits.
     343             : 
     344       58814 :   if (Operation > ISD::SETTRUE2)
     345         280 :     Operation &= ~8;  // Don't let N and U bits get set.
     346             : 
     347       58814 :   return ISD::CondCode(Operation);
     348             : }
     349             : 
     350             : /// For an integer comparison, return 1 if the comparison is a signed operation
     351             : /// and 2 if the result is an unsigned comparison. Return zero if the operation
     352             : /// does not depend on the sign of the input (setne and seteq).
     353             : static int isSignedOp(ISD::CondCode Opcode) {
     354             :   switch (Opcode) {
     355           0 :   default: llvm_unreachable("Illegal integer setcc operation!");
     356             :   case ISD::SETEQ:
     357             :   case ISD::SETNE: return 0;
     358             :   case ISD::SETLT:
     359             :   case ISD::SETLE:
     360             :   case ISD::SETGT:
     361             :   case ISD::SETGE: return 1;
     362             :   case ISD::SETULT:
     363             :   case ISD::SETULE:
     364             :   case ISD::SETUGT:
     365             :   case ISD::SETUGE: return 2;
     366             :   }
     367             : }
     368             : 
     369          37 : ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
     370             :                                        bool IsInteger) {
     371          56 :   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
     372             :     // Cannot fold a signed integer setcc with an unsigned integer setcc.
     373             :     return ISD::SETCC_INVALID;
     374             : 
     375          32 :   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
     376             : 
     377             :   // If the N and U bits get set, then the resultant comparison DOES suddenly
     378             :   // care about orderedness, and it is true when ordered.
     379          32 :   if (Op > ISD::SETTRUE2)
     380          17 :     Op &= ~16;     // Clear the U bit if the N bit is set.
     381             : 
     382             :   // Canonicalize illegal integer setcc's.
     383          32 :   if (IsInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
     384             :     Op = ISD::SETNE;
     385             : 
     386             :   return ISD::CondCode(Op);
     387             : }
     388             : 
     389          33 : ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
     390             :                                         bool IsInteger) {
     391          34 :   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
     392             :     // Cannot fold a signed setcc with an unsigned setcc.
     393             :     return ISD::SETCC_INVALID;
     394             : 
     395             :   // Combine all of the condition bits.
     396          33 :   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
     397             : 
     398             :   // Canonicalize illegal integer setcc's.
     399          33 :   if (IsInteger) {
     400           1 :     switch (Result) {
     401             :     default: break;
     402           0 :     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
     403           1 :     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
     404           1 :     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
     405           0 :     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
     406           0 :     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
     407             :     }
     408             :   }
     409             : 
     410             :   return Result;
     411             : }
     412             : 
     413             : //===----------------------------------------------------------------------===//
     414             : //                           SDNode Profile Support
     415             : //===----------------------------------------------------------------------===//
     416             : 
     417             : /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
     418             : static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
     419    67623132 :   ID.AddInteger(OpC);
     420             : }
     421             : 
     422             : /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
     423             : /// solely with their pointer.
     424             : static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
     425    67623132 :   ID.AddPointer(VTList.VTs);
     426             : }
     427             : 
     428             : /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
     429    33256903 : static void AddNodeIDOperands(FoldingSetNodeID &ID,
     430             :                               ArrayRef<SDValue> Ops) {
     431   117010661 :   for (auto& Op : Ops) {
     432    41876879 :     ID.AddPointer(Op.getNode());
     433    41876879 :     ID.AddInteger(Op.getResNo());
     434             :   }
     435    33256903 : }
     436             : 
     437             : /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
     438    34366230 : static void AddNodeIDOperands(FoldingSetNodeID &ID,
     439             :                               ArrayRef<SDUse> Ops) {
     440   141728364 :   for (auto& Op : Ops) {
     441    53681067 :     ID.AddPointer(Op.getNode());
     442    53681067 :     ID.AddInteger(Op.getResNo());
     443             :   }
     444    34366230 : }
     445             : 
     446    33256902 : static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
     447             :                           SDVTList VTList, ArrayRef<SDValue> OpList) {
     448    33256902 :   AddNodeIDOpcode(ID, OpC);
     449    33256902 :   AddNodeIDValueTypes(ID, VTList);
     450    33256903 :   AddNodeIDOperands(ID, OpList);
     451    33256903 : }
     452             : 
     453             : /// If this is an SDNode with special info, add this info to the NodeID data.
     454    34523288 : static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
     455    69046576 :   switch (N->getOpcode()) {
     456           0 :   case ISD::TargetExternalSymbol:
     457             :   case ISD::ExternalSymbol:
     458             :   case ISD::MCSymbol:
     459           0 :     llvm_unreachable("Should only be used on nodes with operands");
     460             :   default: break;  // Normal nodes don't need extra info.
     461             :   case ISD::TargetConstant:
     462             :   case ISD::Constant: {
     463             :     const ConstantSDNode *C = cast<ConstantSDNode>(N);
     464     8314615 :     ID.AddPointer(C->getConstantIntValue());
     465             :     ID.AddBoolean(C->isOpaque());
     466             :     break;
     467             :   }
     468             :   case ISD::TargetConstantFP:
     469             :   case ISD::ConstantFP:
     470       25900 :     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
     471       25900 :     break;
     472             :   case ISD::TargetGlobalAddress:
     473             :   case ISD::GlobalAddress:
     474             :   case ISD::TargetGlobalTLSAddress:
     475             :   case ISD::GlobalTLSAddress: {
     476             :     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
     477     1488059 :     ID.AddPointer(GA->getGlobal());
     478     1488059 :     ID.AddInteger(GA->getOffset());
     479     1488059 :     ID.AddInteger(GA->getTargetFlags());
     480     1488059 :     break;
     481             :   }
     482             :   case ISD::BasicBlock:
     483      125341 :     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
     484      125341 :     break;
     485             :   case ISD::Register:
     486     3964159 :     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
     487     3964159 :     break;
     488             :   case ISD::RegisterMask:
     489      167734 :     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
     490      167734 :     break;
     491             :   case ISD::SRCVALUE:
     492         428 :     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
     493         428 :     break;
     494             :   case ISD::FrameIndex:
     495             :   case ISD::TargetFrameIndex:
     496      319633 :     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
     497      319633 :     break;
     498             :   case ISD::JumpTable:
     499             :   case ISD::TargetJumpTable:
     500         167 :     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
     501         167 :     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
     502         167 :     break;
     503             :   case ISD::ConstantPool:
     504             :   case ISD::TargetConstantPool: {
     505             :     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
     506       24042 :     ID.AddInteger(CP->getAlignment());
     507       48084 :     ID.AddInteger(CP->getOffset());
     508       24042 :     if (CP->isMachineConstantPoolEntry())
     509          76 :       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
     510             :     else
     511       23966 :       ID.AddPointer(CP->getConstVal());
     512       24042 :     ID.AddInteger(CP->getTargetFlags());
     513       24042 :     break;
     514             :   }
     515             :   case ISD::TargetIndex: {
     516             :     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
     517           0 :     ID.AddInteger(TI->getIndex());
     518           0 :     ID.AddInteger(TI->getOffset());
     519           0 :     ID.AddInteger(TI->getTargetFlags());
     520           0 :     break;
     521             :   }
     522             :   case ISD::LOAD: {
     523             :     const LoadSDNode *LD = cast<LoadSDNode>(N);
     524     1273599 :     ID.AddInteger(LD->getMemoryVT().getRawBits());
     525     1273599 :     ID.AddInteger(LD->getRawSubclassData());
     526     2547198 :     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
     527     1273599 :     break;
     528             :   }
     529             :   case ISD::STORE: {
     530             :     const StoreSDNode *ST = cast<StoreSDNode>(N);
     531     2636040 :     ID.AddInteger(ST->getMemoryVT().getRawBits());
     532     2636040 :     ID.AddInteger(ST->getRawSubclassData());
     533     5272080 :     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
     534     2636040 :     break;
     535             :   }
     536             :   case ISD::ATOMIC_CMP_SWAP:
     537             :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     538             :   case ISD::ATOMIC_SWAP:
     539             :   case ISD::ATOMIC_LOAD_ADD:
     540             :   case ISD::ATOMIC_LOAD_SUB:
     541             :   case ISD::ATOMIC_LOAD_AND:
     542             :   case ISD::ATOMIC_LOAD_CLR:
     543             :   case ISD::ATOMIC_LOAD_OR:
     544             :   case ISD::ATOMIC_LOAD_XOR:
     545             :   case ISD::ATOMIC_LOAD_NAND:
     546             :   case ISD::ATOMIC_LOAD_MIN:
     547             :   case ISD::ATOMIC_LOAD_MAX:
     548             :   case ISD::ATOMIC_LOAD_UMIN:
     549             :   case ISD::ATOMIC_LOAD_UMAX:
     550             :   case ISD::ATOMIC_LOAD:
     551             :   case ISD::ATOMIC_STORE: {
     552             :     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
     553       10414 :     ID.AddInteger(AT->getMemoryVT().getRawBits());
     554       10414 :     ID.AddInteger(AT->getRawSubclassData());
     555       20828 :     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
     556       10414 :     break;
     557             :   }
     558             :   case ISD::PREFETCH: {
     559             :     const MemSDNode *PF = cast<MemSDNode>(N);
     560         512 :     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
     561         256 :     break;
     562             :   }
     563             :   case ISD::VECTOR_SHUFFLE: {
     564             :     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
     565      610125 :     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
     566      566200 :          i != e; ++i)
     567     1044550 :       ID.AddInteger(SVN->getMaskElt(i));
     568             :     break;
     569             :   }
     570             :   case ISD::TargetBlockAddress:
     571             :   case ISD::BlockAddress: {
     572             :     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
     573          72 :     ID.AddPointer(BA->getBlockAddress());
     574          72 :     ID.AddInteger(BA->getOffset());
     575          72 :     ID.AddInteger(BA->getTargetFlags());
     576          72 :     break;
     577             :   }
     578             :   } // end switch (N->getOpcode())
     579             : 
     580             :   // Target specific memory nodes could also have address spaces to check.
     581    34523288 :   if (N->isTargetMemoryOpcode())
     582       17082 :     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
     583    34523288 : }
     584             : 
     585             : /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
     586             : /// data.
     587    34366230 : static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
     588    34366230 :   AddNodeIDOpcode(ID, N->getOpcode());
     589             :   // Add the return value info.
     590    34366230 :   AddNodeIDValueTypes(ID, N->getVTList());
     591             :   // Add the operand info.
     592    34366230 :   AddNodeIDOperands(ID, N->ops());
     593             : 
     594             :   // Handle SDNode leafs with special info.
     595    34366230 :   AddNodeIDCustom(ID, N);
     596    34366230 : }
     597             : 
     598             : //===----------------------------------------------------------------------===//
     599             : //                              SelectionDAG Class
     600             : //===----------------------------------------------------------------------===//
     601             : 
     602             : /// doNotCSE - Return true if CSE should not be performed for this node.
     603     6602367 : static bool doNotCSE(SDNode *N) {
     604     6602367 :   if (N->getValueType(0) == MVT::Glue)
     605             :     return true; // Never CSE anything that produces a flag.
     606             : 
     607    13203598 :   switch (N->getOpcode()) {
     608             :   default: break;
     609             :   case ISD::HANDLENODE:
     610             :   case ISD::EH_LABEL:
     611             :     return true;   // Never CSE these nodes.
     612             :   }
     613             : 
     614             :   // Check that remaining values produced are not flags.
     615    15208884 :   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
     616     1864424 :     if (N->getValueType(i) == MVT::Glue)
     617             :       return true; // Never CSE anything that produces a flag.
     618             : 
     619             :   return false;
     620             : }
     621             : 
     622             : /// RemoveDeadNodes - This method deletes all unreachable nodes in the
     623             : /// SelectionDAG.
     624     1532412 : void SelectionDAG::RemoveDeadNodes() {
     625             :   // Create a dummy node (which is not added to allnodes), that adds a reference
     626             :   // to the root node, preventing it from being deleted.
     627     3064824 :   HandleSDNode Dummy(getRoot());
     628             : 
     629             :   SmallVector<SDNode*, 128> DeadNodes;
     630             : 
     631             :   // Add all obviously-dead nodes to the DeadNodes worklist.
     632    48298665 :   for (SDNode &Node : allnodes())
     633    46766253 :     if (Node.use_empty())
     634      671463 :       DeadNodes.push_back(&Node);
     635             : 
     636     1532412 :   RemoveDeadNodes(DeadNodes);
     637             : 
     638             :   // If the root changed (e.g. it was a dead load, update the root).
     639     1532412 :   setRoot(Dummy.getValue());
     640     1532412 : }
     641             : 
     642             : /// RemoveDeadNodes - This method deletes the unreachable nodes in the
     643             : /// given list, and any nodes that become unreachable as a result.
     644     4304038 : void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
     645             : 
     646             :   // Process the worklist, deleting the nodes and adding their uses to the
     647             :   // worklist.
     648     8845603 :   while (!DeadNodes.empty()) {
     649             :     SDNode *N = DeadNodes.pop_back_val();
     650             :     // Skip to next node if we've already managed to delete the node. This could
     651             :     // happen if replacing a node causes a node previously added to the node to
     652             :     // be deleted.
     653     4541565 :     if (N->getOpcode() == ISD::DELETED_NODE)
     654         134 :       continue;
     655             : 
     656     9660211 :     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
     657     5118780 :       DUL->NodeDeleted(N, nullptr);
     658             : 
     659             :     // Take the node out of the appropriate CSE map.
     660     4541431 :     RemoveNodeFromCSEMaps(N);
     661             : 
     662             :     // Next, brutally remove the operand list.  This is safe to do, as there are
     663             :     // no cycles in the graph.
     664    22286798 :     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
     665     6601968 :       SDUse &Use = *I++;
     666     6601968 :       SDNode *Operand = Use.getNode();
     667     6601968 :       Use.set(SDValue());
     668             : 
     669             :       // Now that we removed this operand, see if there are no uses of it left.
     670     6601968 :       if (Operand->use_empty())
     671     2230201 :         DeadNodes.push_back(Operand);
     672             :     }
     673             : 
     674     4541431 :     DeallocateNode(N);
     675             :   }
     676     4304038 : }
     677             : 
     678      418839 : void SelectionDAG::RemoveDeadNode(SDNode *N){
     679             :   SmallVector<SDNode*, 16> DeadNodes(1, N);
     680             : 
     681             :   // Create a dummy node that adds a reference to the root node, preventing
     682             :   // it from being deleted.  (This matters if the root is an operand of the
     683             :   // dead node.)
     684      837678 :   HandleSDNode Dummy(getRoot());
     685             : 
     686      418839 :   RemoveDeadNodes(DeadNodes);
     687      418839 : }
     688             : 
     689     6011030 : void SelectionDAG::DeleteNode(SDNode *N) {
     690             :   // First take this out of the appropriate CSE map.
     691     6011030 :   RemoveNodeFromCSEMaps(N);
     692             : 
     693             :   // Finally, remove uses due to operands of this node, remove from the
     694             :   // AllNodes list, and delete the node.
     695     6011030 :   DeleteNodeNotInCSEMaps(N);
     696     6011030 : }
     697             : 
     698     6025220 : void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
     699             :   assert(N->getIterator() != AllNodes.begin() &&
     700             :          "Cannot delete the entry node!");
     701             :   assert(N->use_empty() && "Cannot delete a node that is not dead!");
     702             : 
     703             :   // Drop all of the operands and decrement used node's use counts.
     704     6025220 :   N->DropOperands();
     705             : 
     706     6025220 :   DeallocateNode(N);
     707     6025220 : }
     708             : 
     709    18986094 : void SDDbgInfo::erase(const SDNode *Node) {
     710    18986094 :   DbgValMapType::iterator I = DbgValMap.find(Node);
     711    18986094 :   if (I == DbgValMap.end())
     712    18977310 :     return;
     713       46362 :   for (auto &Val: I->second)
     714       18789 :     Val->setIsInvalidated();
     715             :   DbgValMap.erase(I);
     716             : }
     717             : 
     718    18986094 : void SelectionDAG::DeallocateNode(SDNode *N) {
     719             :   // If we have operands, deallocate them.
     720    18986094 :   removeOperands(N);
     721             : 
     722             :   NodeAllocator.Deallocate(AllNodes.remove(N));
     723             : 
     724             :   // Set the opcode to DELETED_NODE to help catch bugs when node
     725             :   // memory is reallocated.
     726             :   // FIXME: There are places in SDag that have grown a dependency on the opcode
     727             :   // value in the released node.
     728             :   __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
     729    18986094 :   N->NodeType = ISD::DELETED_NODE;
     730             : 
     731             :   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
     732             :   // them and forget about that node.
     733    18986094 :   DbgInfo->erase(N);
     734    18986094 : }
     735             : 
     736             : #ifndef NDEBUG
     737             : /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
     738             : static void VerifySDNode(SDNode *N) {
     739             :   switch (N->getOpcode()) {
     740             :   default:
     741             :     break;
     742             :   case ISD::BUILD_PAIR: {
     743             :     EVT VT = N->getValueType(0);
     744             :     assert(N->getNumValues() == 1 && "Too many results!");
     745             :     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
     746             :            "Wrong return type!");
     747             :     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
     748             :     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
     749             :            "Mismatched operand types!");
     750             :     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
     751             :            "Wrong operand type!");
     752             :     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
     753             :            "Wrong return type size");
     754             :     break;
     755             :   }
     756             :   case ISD::BUILD_VECTOR: {
     757             :     assert(N->getNumValues() == 1 && "Too many results!");
     758             :     assert(N->getValueType(0).isVector() && "Wrong return type!");
     759             :     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
     760             :            "Wrong number of operands!");
     761             :     EVT EltVT = N->getValueType(0).getVectorElementType();
     762             :     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
     763             :       assert((I->getValueType() == EltVT ||
     764             :              (EltVT.isInteger() && I->getValueType().isInteger() &&
     765             :               EltVT.bitsLE(I->getValueType()))) &&
     766             :             "Wrong operand type!");
     767             :       assert(I->getValueType() == N->getOperand(0).getValueType() &&
     768             :              "Operands must all have the same type");
     769             :     }
     770             :     break;
     771             :   }
     772             :   }
     773             : }
     774             : #endif // NDEBUG
     775             : 
     776             : /// \brief Insert a newly allocated node into the DAG.
     777             : ///
     778             : /// Handles insertion into the all nodes list and CSE map, as well as
     779             : /// verification and other common operations when a new node is allocated.
     780    19317713 : void SelectionDAG::InsertNode(SDNode *N) {
     781             :   AllNodes.push_back(N);
     782             : #ifndef NDEBUG
     783             :   N->PersistentId = NextPersistentId++;
     784             :   VerifySDNode(N);
     785             : #endif
     786    19317713 : }
     787             : 
     788             : /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
     789             : /// correspond to it.  This is useful when we're about to delete or repurpose
     790             : /// the node.  We don't want future request for structurally identical nodes
     791             : /// to return N anymore.
     792    19930524 : bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
     793             :   bool Erased = false;
     794    39861048 :   switch (N->getOpcode()) {
     795             :   case ISD::HANDLENODE: return false;  // noop.
     796             :   case ISD::CONDCODE:
     797             :     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
     798             :            "Cond code doesn't exist!");
     799      217470 :     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
     800      108735 :     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
     801      108735 :     break;
     802        5846 :   case ISD::ExternalSymbol:
     803       11692 :     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
     804        5846 :     break;
     805             :   case ISD::TargetExternalSymbol: {
     806             :     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
     807         246 :     Erased = TargetExternalSymbols.erase(
     808         492 :                std::pair<std::string,unsigned char>(ESN->getSymbol(),
     809         246 :                                                     ESN->getTargetFlags()));
     810         246 :     break;
     811             :   }
     812             :   case ISD::MCSymbol: {
     813             :     auto *MCSN = cast<MCSymbolSDNode>(N);
     814           0 :     Erased = MCSymbols.erase(MCSN->getMCSymbol());
     815           0 :     break;
     816             :   }
     817             :   case ISD::VALUETYPE: {
     818       41762 :     EVT VT = cast<VTSDNode>(N)->getVT();
     819       41762 :     if (VT.isExtended()) {
     820        1558 :       Erased = ExtendedValueTypeNodes.erase(VT);
     821             :     } else {
     822       80408 :       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
     823       40204 :       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
     824             :     }
     825             :     break;
     826             :   }
     827    19740863 :   default:
     828             :     // Remove it from the CSE Map.
     829             :     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
     830             :     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
     831             :     Erased = CSEMap.RemoveNode(N);
     832    19740863 :     break;
     833             :   }
     834             : #ifndef NDEBUG
     835             :   // Verify that the node was actually in one of the CSE maps, unless it has a
     836             :   // flag result (which cannot be CSE'd) or is one of the special cases that are
     837             :   // not subject to CSE.
     838             :   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
     839             :       !N->isMachineOpcode() && !doNotCSE(N)) {
     840             :     N->dump(this);
     841             :     dbgs() << "\n";
     842             :     llvm_unreachable("Node is not in map!");
     843             :   }
     844             : #endif
     845             :   return Erased;
     846             : }
     847             : 
     848             : /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
     849             : /// maps and modified in place. Add it back to the CSE maps, unless an identical
     850             : /// node already exists, in which case transfer all its users to the existing
     851             : /// node. This transfer can potentially trigger recursive merging.
     852             : void
     853     6434613 : SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
     854             :   // For node types that aren't CSE'd, just act as if no identical node
     855             :   // already exists.
     856     6434613 :   if (!doNotCSE(N)) {
     857             :     SDNode *Existing = CSEMap.GetOrInsertNode(N);
     858     5582960 :     if (Existing != N) {
     859             :       // If there was already an existing matching node, use ReplaceAllUsesWith
     860             :       // to replace the dead one with the existing one.  This can cause
     861             :       // recursive merging of other unrelated nodes down the line.
     862       14190 :       ReplaceAllUsesWith(N, Existing);
     863             : 
     864             :       // N is now dead. Inform the listeners and delete it.
     865       61493 :       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
     866       47303 :         DUL->NodeDeleted(N, Existing);
     867       14190 :       DeleteNodeNotInCSEMaps(N);
     868       14190 :       return;
     869             :     }
     870             :   }
     871             : 
     872             :   // If the node doesn't already exist, we updated it.  Inform listeners.
     873    20861352 :   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
     874    14440929 :     DUL->NodeUpdated(N);
     875             : }
     876             : 
     877             : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
     878             : /// were replaced with those specified.  If this node is never memoized,
     879             : /// return null, otherwise return a pointer to the slot it would take.  If a
     880             : /// node already exists with these operands, the slot will be non-null.
     881        2256 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
     882             :                                            void *&InsertPos) {
     883        2256 :   if (doNotCSE(N))
     884             :     return nullptr;
     885             : 
     886        2256 :   SDValue Ops[] = { Op };
     887             :   FoldingSetNodeID ID;
     888        6768 :   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
     889        2256 :   AddNodeIDCustom(ID, N);
     890        4512 :   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
     891        2256 :   if (Node)
     892           0 :     Node->intersectFlagsWith(N->getFlags());
     893             :   return Node;
     894             : }
     895             : 
     896             : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
     897             : /// were replaced with those specified.  If this node is never memoized,
     898             : /// return null, otherwise return a pointer to the slot it would take.  If a
     899             : /// node already exists with these operands, the slot will be non-null.
     900       56416 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
     901             :                                            SDValue Op1, SDValue Op2,
     902             :                                            void *&InsertPos) {
     903       56416 :   if (doNotCSE(N))
     904             :     return nullptr;
     905             : 
     906       56416 :   SDValue Ops[] = { Op1, Op2 };
     907             :   FoldingSetNodeID ID;
     908      169248 :   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
     909       56416 :   AddNodeIDCustom(ID, N);
     910      112832 :   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
     911       56416 :   if (Node)
     912         699 :     Node->intersectFlagsWith(N->getFlags());
     913             :   return Node;
     914             : }
     915             : 
     916             : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
     917             : /// were replaced with those specified.  If this node is never memoized,
     918             : /// return null, otherwise return a pointer to the slot it would take.  If a
     919             : /// node already exists with these operands, the slot will be non-null.
     920      109082 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
     921             :                                            void *&InsertPos) {
     922      109082 :   if (doNotCSE(N))
     923             :     return nullptr;
     924             : 
     925             :   FoldingSetNodeID ID;
     926      295158 :   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
     927       98386 :   AddNodeIDCustom(ID, N);
     928      196772 :   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
     929       98386 :   if (Node)
     930         186 :     Node->intersectFlagsWith(N->getFlags());
     931             :   return Node;
     932             : }
     933             : 
     934      301919 : unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
     935      301919 :   Type *Ty = VT == MVT::iPTR ?
     936           0 :                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
     937      301919 :                    VT.getTypeForEVT(*getContext());
     938             : 
     939      603838 :   return getDataLayout().getABITypeAlignment(Ty);
     940             : }
     941             : 
     942             : // EntryNode could meaningfully have debug info if we can find it...
     943       20521 : SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
     944             :     : TM(tm), OptLevel(OL),
     945       41042 :       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
     946      123125 :       Root(getEntryNode()) {
     947       20521 :   InsertNode(&EntryNode);
     948       20521 :   DbgInfo = new SDDbgInfo();
     949       20520 : }
     950             : 
     951      166622 : void SelectionDAG::init(MachineFunction &NewMF,
     952             :                         OptimizationRemarkEmitter &NewORE,
     953             :                         Pass *PassPtr, const TargetLibraryInfo *LibraryInfo) {
     954      166622 :   MF = &NewMF;
     955      166622 :   SDAGISelPass = PassPtr;
     956      166622 :   ORE = &NewORE;
     957      166622 :   TLI = getSubtarget().getTargetLowering();
     958      333244 :   TSI = getSubtarget().getSelectionDAGInfo();
     959      166622 :   LibInfo = LibraryInfo;
     960      166622 :   Context = &MF->getFunction().getContext();
     961      166622 : }
     962             : 
     963      102053 : SelectionDAG::~SelectionDAG() {
     964             :   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
     965       20410 :   allnodes_clear();
     966       20411 :   OperandRecycler.clear(OperandAllocator);
     967       20411 :   delete DbgInfo;
     968       20411 : }
     969             : 
     970      330604 : void SelectionDAG::allnodes_clear() {
     971             :   assert(&*AllNodes.begin() == &EntryNode);
     972             :   AllNodes.remove(AllNodes.begin());
     973    17169488 :   while (!AllNodes.empty())
     974     8419443 :     DeallocateNode(&AllNodes.front());
     975             : #ifndef NDEBUG
     976             :   NextPersistentId = 0;
     977             : #endif
     978      330603 : }
     979             : 
     980     5403649 : SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
     981             :                                           void *&InsertPos) {
     982             :   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
     983     5403649 :   if (N) {
     984     6525216 :     switch (N->getOpcode()) {
     985             :     default: break;
     986           0 :     case ISD::Constant:
     987             :     case ISD::ConstantFP:
     988           0 :       llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
     989             :                        "debug location.  Use another overload.");
     990             :     }
     991             :   }
     992     5403649 :   return N;
     993             : }
     994             : 
     995    27853786 : SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
     996             :                                           const SDLoc &DL, void *&InsertPos) {
     997             :   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
     998    27853787 :   if (N) {
     999    19061084 :     switch (N->getOpcode()) {
    1000             :     case ISD::Constant:
    1001             :     case ISD::ConstantFP:
    1002             :       // Erase debug location from the node if the node is used at several
    1003             :       // different places. Do not propagate one location to all uses as it
    1004             :       // will cause a worse single stepping debugging experience.
    1005     2386163 :       if (N->getDebugLoc() != DL.getDebugLoc())
    1006     1199502 :         N->setDebugLoc(DebugLoc());
    1007             :       break;
    1008     7144379 :     default:
    1009             :       // When the node's point of use is located earlier in the instruction
    1010             :       // sequence than its prior point of use, update its debug info to the
    1011             :       // earlier location.
    1012    14288758 :       if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
    1013       80556 :         N->setDebugLoc(DL.getDebugLoc());
    1014             :       break;
    1015             :     }
    1016             :   }
    1017    27853787 :   return N;
    1018             : }
    1019             : 
    1020      310191 : void SelectionDAG::clear() {
    1021      310191 :   allnodes_clear();
    1022      310192 :   OperandRecycler.clear(OperandAllocator);
    1023      310192 :   OperandAllocator.Reset();
    1024      310191 :   CSEMap.clear();
    1025             : 
    1026             :   ExtendedValueTypeNodes.clear();
    1027      310192 :   ExternalSymbols.clear();
    1028             :   TargetExternalSymbols.clear();
    1029      310192 :   MCSymbols.clear();
    1030             :   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
    1031             :             static_cast<CondCodeSDNode*>(nullptr));
    1032             :   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
    1033             :             static_cast<SDNode*>(nullptr));
    1034             : 
    1035      310191 :   EntryNode.UseList = nullptr;
    1036      310191 :   InsertNode(&EntryNode);
    1037      310192 :   Root = getEntryNode();
    1038      310192 :   DbgInfo->clear();
    1039      310191 : }
    1040             : 
    1041          19 : SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
    1042          38 :   return VT.bitsGT(Op.getValueType())
    1043          19 :              ? getNode(ISD::FP_EXTEND, DL, VT, Op)
    1044          57 :              : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
    1045             : }
    1046             : 
    1047       39370 : SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
    1048       78740 :   return VT.bitsGT(Op.getValueType()) ?
    1049       39370 :     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
    1050      118110 :     getNode(ISD::TRUNCATE, DL, VT, Op);
    1051             : }
    1052             : 
    1053      151673 : SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
    1054      303346 :   return VT.bitsGT(Op.getValueType()) ?
    1055      151673 :     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
    1056      455019 :     getNode(ISD::TRUNCATE, DL, VT, Op);
    1057             : }
    1058             : 
    1059      170351 : SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
    1060      340702 :   return VT.bitsGT(Op.getValueType()) ?
    1061      170351 :     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
    1062      511053 :     getNode(ISD::TRUNCATE, DL, VT, Op);
    1063             : }
    1064             : 
    1065         806 : SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
    1066             :                                         EVT OpVT) {
    1067        1612 :   if (VT.bitsLE(Op.getValueType()))
    1068         394 :     return getNode(ISD::TRUNCATE, SL, VT, Op);
    1069             : 
    1070         412 :   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
    1071         412 :   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
    1072             : }
    1073             : 
    1074      103281 : SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
    1075             :   assert(!VT.isVector() &&
    1076             :          "getZeroExtendInReg should use the vector element type instead of "
    1077             :          "the vector type!");
    1078      309843 :   if (Op.getValueType().getScalarType() == VT) return Op;
    1079      102515 :   unsigned BitWidth = Op.getScalarValueSizeInBits();
    1080             :   APInt Imm = APInt::getLowBitsSet(BitWidth,
    1081      102515 :                                    VT.getSizeInBits());
    1082             :   return getNode(ISD::AND, DL, Op.getValueType(), Op,
    1083      307545 :                  getConstant(Imm, DL, Op.getValueType()));
    1084             : }
    1085             : 
    1086         265 : SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL,
    1087             :                                               EVT VT) {
    1088             :   assert(VT.isVector() && "This DAG node is restricted to vector types.");
    1089             :   assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
    1090             :          "The sizes of the input and result must match in order to perform the "
    1091             :          "extend in-register.");
    1092             :   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
    1093             :          "The destination vector type must have fewer lanes than the input.");
    1094         265 :   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
    1095             : }
    1096             : 
    1097        1208 : SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL,
    1098             :                                                EVT VT) {
    1099             :   assert(VT.isVector() && "This DAG node is restricted to vector types.");
    1100             :   assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
    1101             :          "The sizes of the input and result must match in order to perform the "
    1102             :          "extend in-register.");
    1103             :   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
    1104             :          "The destination vector type must have fewer lanes than the input.");
    1105        1208 :   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
    1106             : }
    1107             : 
    1108        1785 : SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL,
    1109             :                                                EVT VT) {
    1110             :   assert(VT.isVector() && "This DAG node is restricted to vector types.");
    1111             :   assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
    1112             :          "The sizes of the input and result must match in order to perform the "
    1113             :          "extend in-register.");
    1114             :   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
    1115             :          "The destination vector type must have fewer lanes than the input.");
    1116        1785 :   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
    1117             : }
    1118             : 
    1119             : /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
    1120       25168 : SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
    1121       25168 :   EVT EltVT = VT.getScalarType();
    1122             :   SDValue NegOne =
    1123       50336 :     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
    1124       25168 :   return getNode(ISD::XOR, DL, VT, Val, NegOne);
    1125             : }
    1126             : 
    1127          48 : SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
    1128          48 :   SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
    1129          48 :   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
    1130             : }
    1131             : 
    1132        1142 : SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
    1133             :                                       EVT OpVT) {
    1134        1142 :   if (!V)
    1135         546 :     return getConstant(0, DL, VT);
    1136             : 
    1137         596 :   switch (TLI->getBooleanContents(OpVT)) {
    1138         512 :   case TargetLowering::ZeroOrOneBooleanContent:
    1139             :   case TargetLowering::UndefinedBooleanContent:
    1140         512 :     return getConstant(1, DL, VT);
    1141          84 :   case TargetLowering::ZeroOrNegativeOneBooleanContent:
    1142          84 :     return getAllOnesConstant(DL, VT);
    1143             :   }
    1144           0 :   llvm_unreachable("Unexpected boolean content enum!");
    1145             : }
    1146             : 
    1147     6989908 : SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
    1148             :                                   bool isT, bool isO) {
    1149     6989908 :   EVT EltVT = VT.getScalarType();
    1150             :   assert((EltVT.getSizeInBits() >= 64 ||
    1151             :          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
    1152             :          "getConstant with a uint64_t value that doesn't fit in the type!");
    1153    20969724 :   return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
    1154             : }
    1155             : 
    1156     8752656 : SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
    1157             :                                   bool isT, bool isO) {
    1158     8752656 :   return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
    1159             : }
    1160             : 
    1161     9389461 : SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
    1162             :                                   EVT VT, bool isT, bool isO) {
    1163             :   assert(VT.isInteger() && "Cannot create FP integer constant!");
    1164             : 
    1165     9389461 :   EVT EltVT = VT.getScalarType();
    1166             :   const ConstantInt *Elt = &Val;
    1167             : 
    1168             :   // In some cases the vector type is legal but the element type is illegal and
    1169             :   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
    1170             :   // inserted value (the type does not need to match the vector element type).
    1171             :   // Any extra bits introduced will be truncated away.
    1172     9532635 :   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
    1173             :       TargetLowering::TypePromoteInteger) {
    1174       12866 :    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
    1175        6433 :    APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
    1176        6433 :    Elt = ConstantInt::get(*getContext(), NewVal);
    1177             :   }
    1178             :   // In other cases the element type is illegal and needs to be expanded, for
    1179             :   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
    1180             :   // the value into n parts and use a vector type with n-times the elements.
    1181             :   // Then bitcast to the type requested.
    1182             :   // Legalizing constants too early makes the DAGCombiner's job harder so we
    1183             :   // only legalize if the DAG tells us we must produce legal types.
    1184    14901239 :   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
    1185       46912 :            TLI->getTypeAction(*getContext(), EltVT) ==
    1186             :            TargetLowering::TypeExpandInteger) {
    1187             :     const APInt &NewVal = Elt->getValue();
    1188         546 :     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
    1189         273 :     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
    1190         273 :     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
    1191         273 :     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
    1192             : 
    1193             :     // Check the temporary vector is the correct size. If this fails then
    1194             :     // getTypeToTransformTo() probably returned a type whose size (in bits)
    1195             :     // isn't a power-of-2 factor of the requested type size.
    1196             :     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
    1197             : 
    1198             :     SmallVector<SDValue, 2> EltParts;
    1199        1365 :     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
    1200        1638 :       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
    1201        1092 :                                            .zextOrTrunc(ViaEltSizeInBits), DL,
    1202         546 :                                      ViaEltVT, isT, isO));
    1203             :     }
    1204             : 
    1205             :     // EltParts is currently in little endian order. If we actually want
    1206             :     // big-endian order then reverse it now.
    1207         546 :     if (getDataLayout().isBigEndian())
    1208             :       std::reverse(EltParts.begin(), EltParts.end());
    1209             : 
    1210             :     // The elements must be reversed when the element order is different
    1211             :     // to the endianness of the elements (because the BITCAST is itself a
    1212             :     // vector shuffle in this situation). However, we do not need any code to
    1213             :     // perform this reversal because getConstant() is producing a vector
    1214             :     // splat.
    1215             :     // This situation occurs in MIPS MSA.
    1216             : 
    1217             :     SmallVector<SDValue, 8> Ops;
    1218         881 :     for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    1219         608 :       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
    1220             : 
    1221         273 :     SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
    1222         273 :     return V;
    1223             :   }
    1224             : 
    1225             :   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
    1226             :          "APInt size does not match type size!");
    1227     9389188 :   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
    1228             :   FoldingSetNodeID ID;
    1229     9389188 :   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
    1230     9389188 :   ID.AddPointer(Elt);
    1231             :   ID.AddBoolean(isO);
    1232     9389188 :   void *IP = nullptr;
    1233             :   SDNode *N = nullptr;
    1234     9389188 :   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
    1235     6038313 :     if (!VT.isVector())
    1236     5996021 :       return SDValue(N, 0);
    1237             : 
    1238     3393167 :   if (!N) {
    1239     3350875 :     N = newSDNode<ConstantSDNode>(isT, isO, Elt, DL.getDebugLoc(), EltVT);
    1240     3350875 :     CSEMap.InsertNode(N, IP);
    1241     3350875 :     InsertNode(N);
    1242             :     NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
    1243             :   }
    1244             : 
    1245             :   SDValue Result(N, 0);
    1246     3393167 :   if (VT.isVector())
    1247      142901 :     Result = getSplatBuildVector(VT, DL, Result);
    1248             : 
    1249     3393167 :   return Result;
    1250             : }
    1251             : 
    1252     1019332 : SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
    1253             :                                         bool isTarget) {
    1254     2038664 :   return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
    1255             : }
    1256             : 
    1257       11777 : SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
    1258             :                                     bool isTarget) {
    1259       11777 :   return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
    1260             : }
    1261             : 
    1262       27289 : SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
    1263             :                                     EVT VT, bool isTarget) {
    1264             :   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
    1265             : 
    1266       27289 :   EVT EltVT = VT.getScalarType();
    1267             : 
    1268             :   // Do the map lookup using the actual bit pattern for the floating point
    1269             :   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
    1270             :   // we don't have issues with SNANs.
    1271       27289 :   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
    1272             :   FoldingSetNodeID ID;
    1273       27289 :   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
    1274       27289 :   ID.AddPointer(&V);
    1275       27289 :   void *IP = nullptr;
    1276             :   SDNode *N = nullptr;
    1277       27289 :   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
    1278        4933 :     if (!VT.isVector())
    1279        4544 :       return SDValue(N, 0);
    1280             : 
    1281       22745 :   if (!N) {
    1282       22356 :     N = newSDNode<ConstantFPSDNode>(isTarget, &V, DL.getDebugLoc(), EltVT);
    1283       22356 :     CSEMap.InsertNode(N, IP);
    1284       22356 :     InsertNode(N);
    1285             :   }
    1286             : 
    1287             :   SDValue Result(N, 0);
    1288       22745 :   if (VT.isVector())
    1289         979 :     Result = getSplatBuildVector(VT, DL, Result);
    1290             :   NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
    1291       22745 :   return Result;
    1292             : }
    1293             : 
    1294        5837 : SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
    1295             :                                     bool isTarget) {
    1296        5837 :   EVT EltVT = VT.getScalarType();
    1297             :   if (EltVT == MVT::f32)
    1298        7772 :     return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
    1299             :   else if (EltVT == MVT::f64)
    1300        3570 :     return getConstantFP(APFloat(Val), DL, VT, isTarget);
    1301             :   else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
    1302             :            EltVT == MVT::f16) {
    1303             :     bool Ignored;
    1304         166 :     APFloat APF = APFloat(Val);
    1305         166 :     APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
    1306             :                 &Ignored);
    1307         166 :     return getConstantFP(APF, DL, VT, isTarget);
    1308             :   } else
    1309           0 :     llvm_unreachable("Unsupported type in getConstantFP");
    1310             : }
    1311             : 
    1312     1703342 : SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
    1313             :                                        EVT VT, int64_t Offset, bool isTargetGA,
    1314             :                                        unsigned char TargetFlags) {
    1315             :   assert((TargetFlags == 0 || isTargetGA) &&
    1316             :          "Cannot set target flags on target-independent globals");
    1317             : 
    1318             :   // Truncate (with sign-extension) the offset value to the pointer size.
    1319     3406684 :   unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
    1320     1703342 :   if (BitWidth < 64)
    1321      870532 :     Offset = SignExtend64(Offset, BitWidth);
    1322             : 
    1323             :   unsigned Opc;
    1324     1703342 :   if (GV->isThreadLocal())
    1325        1725 :     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
    1326             :   else
    1327     1701617 :     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
    1328             : 
    1329             :   FoldingSetNodeID ID;
    1330     1703342 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1331     1703342 :   ID.AddPointer(GV);
    1332     1703342 :   ID.AddInteger(Offset);
    1333     1703342 :   ID.AddInteger(TargetFlags);
    1334     1703342 :   void *IP = nullptr;
    1335     1703342 :   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    1336      265336 :     return SDValue(E, 0);
    1337             : 
    1338             :   auto *N = newSDNode<GlobalAddressSDNode>(
    1339     2876012 :       Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
    1340     1438006 :   CSEMap.InsertNode(N, IP);
    1341     1438006 :     InsertNode(N);
    1342     1438006 :   return SDValue(N, 0);
    1343             : }
    1344             : 
    1345      328004 : SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
    1346      328004 :   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
    1347             :   FoldingSetNodeID ID;
    1348      328004 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1349      328004 :   ID.AddInteger(FI);
    1350      328004 :   void *IP = nullptr;
    1351      328004 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1352       52461 :     return SDValue(E, 0);
    1353             : 
    1354      275543 :   auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
    1355      275543 :   CSEMap.InsertNode(N, IP);
    1356      275543 :   InsertNode(N);
    1357      275543 :   return SDValue(N, 0);
    1358             : }
    1359             : 
    1360         568 : SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
    1361             :                                    unsigned char TargetFlags) {
    1362             :   assert((TargetFlags == 0 || isTarget) &&
    1363             :          "Cannot set target flags on target-independent jump tables");
    1364         568 :   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
    1365             :   FoldingSetNodeID ID;
    1366         568 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1367         568 :   ID.AddInteger(JTI);
    1368         568 :   ID.AddInteger(TargetFlags);
    1369         568 :   void *IP = nullptr;
    1370         568 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1371          46 :     return SDValue(E, 0);
    1372             : 
    1373         522 :   auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
    1374         522 :   CSEMap.InsertNode(N, IP);
    1375         522 :   InsertNode(N);
    1376         522 :   return SDValue(N, 0);
    1377             : }
    1378             : 
    1379       67143 : SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
    1380             :                                       unsigned Alignment, int Offset,
    1381             :                                       bool isTarget,
    1382             :                                       unsigned char TargetFlags) {
    1383             :   assert((TargetFlags == 0 || isTarget) &&
    1384             :          "Cannot set target flags on target-independent globals");
    1385       67143 :   if (Alignment == 0)
    1386       23617 :     Alignment = MF->getFunction().optForSize()
    1387       47234 :                     ? getDataLayout().getABITypeAlignment(C->getType())
    1388       46822 :                     : getDataLayout().getPrefTypeAlignment(C->getType());
    1389       67143 :   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
    1390             :   FoldingSetNodeID ID;
    1391       67143 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1392       67143 :   ID.AddInteger(Alignment);
    1393       67143 :   ID.AddInteger(Offset);
    1394       67143 :   ID.AddPointer(C);
    1395       67143 :   ID.AddInteger(TargetFlags);
    1396       67143 :   void *IP = nullptr;
    1397       67143 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1398        3885 :     return SDValue(E, 0);
    1399             : 
    1400             :   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
    1401       63258 :                                           TargetFlags);
    1402       63258 :   CSEMap.InsertNode(N, IP);
    1403       63258 :   InsertNode(N);
    1404       63258 :   return SDValue(N, 0);
    1405             : }
    1406             : 
    1407         274 : SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
    1408             :                                       unsigned Alignment, int Offset,
    1409             :                                       bool isTarget,
    1410             :                                       unsigned char TargetFlags) {
    1411             :   assert((TargetFlags == 0 || isTarget) &&
    1412             :          "Cannot set target flags on target-independent globals");
    1413         274 :   if (Alignment == 0)
    1414           0 :     Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
    1415         274 :   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
    1416             :   FoldingSetNodeID ID;
    1417         274 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1418         274 :   ID.AddInteger(Alignment);
    1419         274 :   ID.AddInteger(Offset);
    1420         274 :   C->addSelectionDAGCSEId(ID);
    1421         274 :   ID.AddInteger(TargetFlags);
    1422         274 :   void *IP = nullptr;
    1423         274 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1424           0 :     return SDValue(E, 0);
    1425             : 
    1426             :   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
    1427         274 :                                           TargetFlags);
    1428         274 :   CSEMap.InsertNode(N, IP);
    1429         274 :   InsertNode(N);
    1430         274 :   return SDValue(N, 0);
    1431             : }
    1432             : 
    1433           0 : SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
    1434             :                                      unsigned char TargetFlags) {
    1435             :   FoldingSetNodeID ID;
    1436           0 :   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
    1437           0 :   ID.AddInteger(Index);
    1438           0 :   ID.AddInteger(Offset);
    1439           0 :   ID.AddInteger(TargetFlags);
    1440           0 :   void *IP = nullptr;
    1441           0 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1442           0 :     return SDValue(E, 0);
    1443             : 
    1444           0 :   auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
    1445           0 :   CSEMap.InsertNode(N, IP);
    1446           0 :   InsertNode(N);
    1447           0 :   return SDValue(N, 0);
    1448             : }
    1449             : 
    1450      140863 : SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
    1451             :   FoldingSetNodeID ID;
    1452      140863 :   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
    1453      140863 :   ID.AddPointer(MBB);
    1454      140863 :   void *IP = nullptr;
    1455      140863 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1456           5 :     return SDValue(E, 0);
    1457             : 
    1458      140858 :   auto *N = newSDNode<BasicBlockSDNode>(MBB);
    1459      140858 :   CSEMap.InsertNode(N, IP);
    1460      140858 :   InsertNode(N);
    1461      140858 :   return SDValue(N, 0);
    1462             : }
    1463             : 
    1464       77216 : SDValue SelectionDAG::getValueType(EVT VT) {
    1465      152426 :   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
    1466       75210 :       ValueTypeNodes.size())
    1467        4116 :     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
    1468             : 
    1469      154432 :   SDNode *&N = VT.isExtended() ?
    1470       77216 :     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
    1471             : 
    1472       77216 :   if (N) return SDValue(N, 0);
    1473       41762 :   N = newSDNode<VTSDNode>(VT);
    1474       41762 :   InsertNode(N);
    1475       41762 :   return SDValue(N, 0);
    1476             : }
    1477             : 
    1478        7943 : SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
    1479        7943 :   SDNode *&N = ExternalSymbols[Sym];
    1480        7943 :   if (N) return SDValue(N, 0);
    1481        5847 :   N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
    1482        5847 :   InsertNode(N);
    1483        5847 :   return SDValue(N, 0);
    1484             : }
    1485             : 
    1486          69 : SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
    1487          69 :   SDNode *&N = MCSymbols[Sym];
    1488          69 :   if (N)
    1489          17 :     return SDValue(N, 0);
    1490          52 :   N = newSDNode<MCSymbolSDNode>(Sym, VT);
    1491          52 :   InsertNode(N);
    1492          52 :   return SDValue(N, 0);
    1493             : }
    1494             : 
    1495       22443 : SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
    1496             :                                               unsigned char TargetFlags) {
    1497             :   SDNode *&N =
    1498       22443 :     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
    1499       22443 :                                                                TargetFlags)];
    1500       22443 :   if (N) return SDValue(N, 0);
    1501       16237 :   N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
    1502       16237 :   InsertNode(N);
    1503       16237 :   return SDValue(N, 0);
    1504             : }
    1505             : 
    1506      158064 : SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
    1507      316128 :   if ((unsigned)Cond >= CondCodeNodes.size())
    1508        8410 :     CondCodeNodes.resize(Cond+1);
    1509             : 
    1510      316128 :   if (!CondCodeNodes[Cond]) {
    1511      108735 :     auto *N = newSDNode<CondCodeSDNode>(Cond);
    1512      217470 :     CondCodeNodes[Cond] = N;
    1513      108735 :     InsertNode(N);
    1514             :   }
    1515             : 
    1516      316128 :   return SDValue(CondCodeNodes[Cond], 0);
    1517             : }
    1518             : 
    1519             : /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
    1520             : /// point at N1 to point at N2 and indices that point at N2 to point at N1.
    1521             : static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
    1522             :   std::swap(N1, N2);
    1523             :   ShuffleVectorSDNode::commuteMask(M);
    1524             : }
    1525             : 
    1526       88941 : SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
    1527             :                                        SDValue N2, ArrayRef<int> Mask) {
    1528             :   assert(VT.getVectorNumElements() == Mask.size() &&
    1529             :            "Must have the same number of vector elements as mask elements!");
    1530             :   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
    1531             :          "Invalid VECTOR_SHUFFLE");
    1532             : 
    1533             :   // Canonicalize shuffle undef, undef -> undef
    1534       89199 :   if (N1.isUndef() && N2.isUndef())
    1535          76 :     return getUNDEF(VT);
    1536             : 
    1537             :   // Validate that all indices in Mask are within the range of the elements
    1538             :   // input to the shuffle.
    1539       88865 :   int NElts = Mask.size();
    1540             :   assert(llvm::all_of(Mask,
    1541             :                       [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
    1542             :          "Index out of range");
    1543             : 
    1544             :   // Copy the mask so we can do any needed cleanup.
    1545             :   SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
    1546             : 
    1547             :   // Canonicalize shuffle v, v -> v, undef
    1548             :   if (N1 == N2) {
    1549        5318 :     N2 = getUNDEF(VT);
    1550       84980 :     for (int i = 0; i != NElts; ++i)
    1551       79662 :       if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
    1552             :   }
    1553             : 
    1554             :   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
    1555       88865 :   if (N1.isUndef())
    1556             :     commuteShuffle(N1, N2, MaskVec);
    1557             : 
    1558             :   // If shuffling a splat, try to blend the splat instead. We do this here so
    1559             :   // that even when this arises during lowering we don't have to re-handle it.
    1560        9395 :   auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
    1561             :     BitVector UndefElements;
    1562        9395 :     SDValue Splat = BV->getSplatValue(&UndefElements);
    1563        9395 :     if (!Splat)
    1564             :       return;
    1565             : 
    1566      199078 :     for (int i = 0; i < NElts; ++i) {
    1567      190598 :       if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
    1568       36529 :         continue;
    1569             : 
    1570             :       // If this input comes from undef, mark it as such.
    1571      117625 :       if (UndefElements[MaskVec[i] - Offset]) {
    1572          85 :         MaskVec[i] = -1;
    1573          85 :         continue;
    1574             :       }
    1575             : 
    1576             :       // If we can blend a non-undef lane, use that instead.
    1577      117370 :       if (!UndefElements[i])
    1578       56529 :         MaskVec[i] = i + Offset;
    1579             :     }
    1580       88865 :   };
    1581             :   if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
    1582        4329 :     BlendSplat(N1BV, 0);
    1583             :   if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
    1584        5066 :     BlendSplat(N2BV, NElts);
    1585             : 
    1586             :   // Canonicalize all index into lhs, -> shuffle lhs, undef
    1587             :   // Canonicalize all index into rhs, -> shuffle rhs, undef
    1588             :   bool AllLHS = true, AllRHS = true;
    1589             :   bool N2Undef = N2.isUndef();
    1590     2017723 :   for (int i = 0; i != NElts; ++i) {
    1591     1928858 :     if (MaskVec[i] >= NElts) {
    1592      139455 :       if (N2Undef)
    1593        2880 :         MaskVec[i] = -1;
    1594             :       else
    1595             :         AllLHS = false;
    1596      824974 :     } else if (MaskVec[i] >= 0) {
    1597             :       AllRHS = false;
    1598             :     }
    1599             :   }
    1600       88865 :   if (AllLHS && AllRHS)
    1601          55 :     return getUNDEF(VT);
    1602       88810 :   if (AllLHS && !N2Undef)
    1603        3475 :     N2 = getUNDEF(VT);
    1604       88810 :   if (AllRHS) {
    1605         586 :     N1 = getUNDEF(VT);
    1606             :     commuteShuffle(N1, N2, MaskVec);
    1607             :   }
    1608             :   // Reset our undef status after accounting for the mask.
    1609             :   N2Undef = N2.isUndef();
    1610             :   // Re-check whether both sides ended up undef.
    1611       88810 :   if (N1.isUndef() && N2Undef)
    1612           0 :     return getUNDEF(VT);
    1613             : 
    1614             :   // If Identity shuffle return that node.
    1615             :   bool Identity = true, AllSame = true;
    1616     2017118 :   for (int i = 0; i != NElts; ++i) {
    1617     1928308 :     if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
    1618      964154 :     if (MaskVec[i] != MaskVec[0]) AllSame = false;
    1619             :   }
    1620       88810 :   if (Identity && NElts)
    1621        8726 :     return N1;
    1622             : 
    1623             :   // Shuffling a constant splat doesn't change the result.
    1624       80084 :   if (N2Undef) {
    1625             :     SDValue V = N1;
    1626             : 
    1627             :     // Look through any bitcasts. We check that these don't change the number
    1628             :     // (and size) of elements and just changes their types.
    1629       69898 :     while (V.getOpcode() == ISD::BITCAST)
    1630       12947 :       V = V->getOperand(0);
    1631             : 
    1632             :     // A splat should always show up as a build vector node.
    1633             :     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
    1634             :       BitVector UndefElements;
    1635        1269 :       SDValue Splat = BV->getSplatValue(&UndefElements);
    1636             :       // If this is a splat of an undef, shuffling it is also undef.
    1637        1439 :       if (Splat && Splat.isUndef())
    1638           0 :         return getUNDEF(VT);
    1639             : 
    1640             :       bool SameNumElts =
    1641        1269 :           V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
    1642             : 
    1643             :       // We only have a splat which can skip shuffles if there is a splatted
    1644             :       // value and no undef lanes rearranged by the shuffle.
    1645        1439 :       if (Splat && UndefElements.none()) {
    1646             :         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
    1647             :         // number of elements match or the value splatted is a zero constant.
    1648          43 :         if (SameNumElts)
    1649           4 :           return N1;
    1650             :         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
    1651          48 :           if (C->isNullValue())
    1652          17 :             return N1;
    1653             :       }
    1654             : 
    1655             :       // If the shuffle itself creates a splat, build the vector directly.
    1656        1248 :       if (AllSame && SameNumElts) {
    1657         271 :         EVT BuildVT = BV->getValueType(0);
    1658         271 :         const SDValue &Splatted = BV->getOperand(MaskVec[0]);
    1659         271 :         SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
    1660             : 
    1661             :         // We may have jumped through bitcasts, so the type of the
    1662             :         // BUILD_VECTOR may not match the type of the shuffle.
    1663         271 :         if (BuildVT != VT)
    1664           0 :           NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
    1665         271 :         return NewBV;
    1666             :       }
    1667             :     }
    1668             :   }
    1669             : 
    1670             :   FoldingSetNodeID ID;
    1671       79792 :   SDValue Ops[2] = { N1, N2 };
    1672       79792 :   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
    1673     1881672 :   for (int i = 0; i != NElts; ++i)
    1674     1801880 :     ID.AddInteger(MaskVec[i]);
    1675             : 
    1676       79792 :   void* IP = nullptr;
    1677       79792 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
    1678        1329 :     return SDValue(E, 0);
    1679             : 
    1680             :   // Allocate the mask array for the node out of the BumpPtrAllocator, since
    1681             :   // SDNode doesn't have access to it.  This memory will be "leaked" when
    1682             :   // the node is deallocated, but recovered when the NodeAllocator is released.
    1683       78463 :   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
    1684             :   std::copy(MaskVec.begin(), MaskVec.end(), MaskAlloc);
    1685             : 
    1686       78463 :   auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
    1687       78463 :                                            dl.getDebugLoc(), MaskAlloc);
    1688       78463 :   createOperands(N, Ops);
    1689             : 
    1690       78463 :   CSEMap.InsertNode(N, IP);
    1691       78463 :   InsertNode(N);
    1692             :   SDValue V = SDValue(N, 0);
    1693             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    1694       78463 :   return V;
    1695             : }
    1696             : 
    1697        4248 : SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
    1698        8496 :   EVT VT = SV.getValueType(0);
    1699             :   SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
    1700             :   ShuffleVectorSDNode::commuteMask(MaskVec);
    1701             : 
    1702        4248 :   SDValue Op0 = SV.getOperand(0);
    1703        4248 :   SDValue Op1 = SV.getOperand(1);
    1704       12744 :   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
    1705             : }
    1706             : 
    1707     4604285 : SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
    1708             :   FoldingSetNodeID ID;
    1709     4604285 :   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
    1710     4604285 :   ID.AddInteger(RegNo);
    1711     4604285 :   void *IP = nullptr;
    1712     4604285 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1713     3109722 :     return SDValue(E, 0);
    1714             : 
    1715     1494563 :   auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
    1716     1494563 :   CSEMap.InsertNode(N, IP);
    1717     1494563 :   InsertNode(N);
    1718     1494563 :   return SDValue(N, 0);
    1719             : }
    1720             : 
    1721      194830 : SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
    1722             :   FoldingSetNodeID ID;
    1723      194830 :   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
    1724      194830 :   ID.AddPointer(RegMask);
    1725      194830 :   void *IP = nullptr;
    1726      194830 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1727       96319 :     return SDValue(E, 0);
    1728             : 
    1729       98511 :   auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
    1730       98511 :   CSEMap.InsertNode(N, IP);
    1731       98511 :   InsertNode(N);
    1732       98511 :   return SDValue(N, 0);
    1733             : }
    1734             : 
    1735       54490 : SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
    1736             :                                  MCSymbol *Label) {
    1737       54490 :   return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
    1738             : }
    1739             : 
    1740       54491 : SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
    1741             :                                    SDValue Root, MCSymbol *Label) {
    1742             :   FoldingSetNodeID ID;
    1743       54491 :   SDValue Ops[] = { Root };
    1744       54491 :   AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
    1745       54491 :   ID.AddPointer(Label);
    1746       54491 :   void *IP = nullptr;
    1747       54491 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1748           0 :     return SDValue(E, 0);
    1749             : 
    1750      108982 :   auto *N = newSDNode<LabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label);
    1751       54491 :   createOperands(N, Ops);
    1752             : 
    1753       54491 :   CSEMap.InsertNode(N, IP);
    1754       54491 :   InsertNode(N);
    1755       54491 :   return SDValue(N, 0);
    1756             : }
    1757             : 
    1758         249 : SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
    1759             :                                       int64_t Offset,
    1760             :                                       bool isTarget,
    1761             :                                       unsigned char TargetFlags) {
    1762         249 :   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
    1763             : 
    1764             :   FoldingSetNodeID ID;
    1765         249 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1766         249 :   ID.AddPointer(BA);
    1767         249 :   ID.AddInteger(Offset);
    1768         249 :   ID.AddInteger(TargetFlags);
    1769         249 :   void *IP = nullptr;
    1770         249 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1771           2 :     return SDValue(E, 0);
    1772             : 
    1773         247 :   auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
    1774         247 :   CSEMap.InsertNode(N, IP);
    1775         247 :   InsertNode(N);
    1776         247 :   return SDValue(N, 0);
    1777             : }
    1778             : 
    1779         534 : SDValue SelectionDAG::getSrcValue(const Value *V) {
    1780             :   assert((!V || V->getType()->isPointerTy()) &&
    1781             :          "SrcValue is not a pointer?");
    1782             : 
    1783             :   FoldingSetNodeID ID;
    1784         534 :   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
    1785         534 :   ID.AddPointer(V);
    1786             : 
    1787         534 :   void *IP = nullptr;
    1788         534 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1789         168 :     return SDValue(E, 0);
    1790             : 
    1791         366 :   auto *N = newSDNode<SrcValueSDNode>(V);
    1792         366 :   CSEMap.InsertNode(N, IP);
    1793         366 :   InsertNode(N);
    1794         366 :   return SDValue(N, 0);
    1795             : }
    1796             : 
    1797       12408 : SDValue SelectionDAG::getMDNode(const MDNode *MD) {
    1798             :   FoldingSetNodeID ID;
    1799       12408 :   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
    1800       12408 :   ID.AddPointer(MD);
    1801             : 
    1802       12408 :   void *IP = nullptr;
    1803       12408 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1804           0 :     return SDValue(E, 0);
    1805             : 
    1806       12408 :   auto *N = newSDNode<MDNodeSDNode>(MD);
    1807       12408 :   CSEMap.InsertNode(N, IP);
    1808       12408 :   InsertNode(N);
    1809       12408 :   return SDValue(N, 0);
    1810             : }
    1811             : 
    1812      301593 : SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
    1813          46 :   if (VT == V.getValueType())
    1814       75168 :     return V;
    1815             : 
    1816      452850 :   return getNode(ISD::BITCAST, SDLoc(V), VT, V);
    1817             : }
    1818             : 
    1819         201 : SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
    1820             :                                        unsigned SrcAS, unsigned DestAS) {
    1821         201 :   SDValue Ops[] = {Ptr};
    1822             :   FoldingSetNodeID ID;
    1823         201 :   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
    1824         201 :   ID.AddInteger(SrcAS);
    1825         201 :   ID.AddInteger(DestAS);
    1826             : 
    1827         201 :   void *IP = nullptr;
    1828         201 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
    1829           0 :     return SDValue(E, 0);
    1830             : 
    1831         201 :   auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
    1832         201 :                                            VT, SrcAS, DestAS);
    1833         201 :   createOperands(N, Ops);
    1834             : 
    1835         201 :   CSEMap.InsertNode(N, IP);
    1836         201 :   InsertNode(N);
    1837         201 :   return SDValue(N, 0);
    1838             : }
    1839             : 
    1840             : /// getShiftAmountOperand - Return the specified value casted to
    1841             : /// the target's desired shift amount type.
    1842      264009 : SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
    1843      264009 :   EVT OpTy = Op.getValueType();
    1844      528018 :   EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
    1845      528018 :   if (OpTy == ShTy || OpTy.isVector()) return Op;
    1846             : 
    1847       80874 :   return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
    1848             : }
    1849             : 
    1850          27 : SDValue SelectionDAG::expandVAArg(SDNode *Node) {
    1851             :   SDLoc dl(Node);
    1852          27 :   const TargetLowering &TLI = getTargetLoweringInfo();
    1853          54 :   const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
    1854          54 :   EVT VT = Node->getValueType(0);
    1855          27 :   SDValue Tmp1 = Node->getOperand(0);
    1856          27 :   SDValue Tmp2 = Node->getOperand(1);
    1857          27 :   unsigned Align = Node->getConstantOperandVal(3);
    1858             : 
    1859             :   SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
    1860          54 :                                Tmp2, MachinePointerInfo(V));
    1861          27 :   SDValue VAList = VAListLoad;
    1862             : 
    1863          27 :   if (Align > TLI.getMinStackArgumentAlignment()) {
    1864             :     assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
    1865             : 
    1866          11 :     VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
    1867          22 :                      getConstant(Align - 1, dl, VAList.getValueType()));
    1868             : 
    1869          11 :     VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
    1870          22 :                      getConstant(-(int64_t)Align, dl, VAList.getValueType()));
    1871             :   }
    1872             : 
    1873             :   // Increment the pointer, VAList, to the next vaarg
    1874          27 :   Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
    1875          27 :                  getConstant(getDataLayout().getTypeAllocSize(
    1876          27 :                                                VT.getTypeForEVT(*getContext())),
    1877          54 :                              dl, VAList.getValueType()));
    1878             :   // Store the incremented VAList to the legalized pointer
    1879          27 :   Tmp1 =
    1880          54 :       getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
    1881             :   // Load the actual argument out of the pointer VAList
    1882          54 :   return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
    1883             : }
    1884             : 
    1885           5 : SDValue SelectionDAG::expandVACopy(SDNode *Node) {
    1886             :   SDLoc dl(Node);
    1887             :   const TargetLowering &TLI = getTargetLoweringInfo();
    1888             :   // This defaults to loading a pointer from the input and storing it to the
    1889             :   // output, returning the chain.
    1890          10 :   const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
    1891           5 :   const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
    1892             :   SDValue Tmp1 =
    1893             :       getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
    1894          10 :               Node->getOperand(2), MachinePointerInfo(VS));
    1895           5 :   return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
    1896          10 :                   MachinePointerInfo(VD));
    1897             : }
    1898             : 
    1899        2607 : SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
    1900        2607 :   MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
    1901             :   unsigned ByteSize = VT.getStoreSize();
    1902        2607 :   Type *Ty = VT.getTypeForEVT(*getContext());
    1903             :   unsigned StackAlign =
    1904        7821 :       std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
    1905             : 
    1906        2607 :   int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
    1907        7821 :   return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
    1908             : }
    1909             : 
    1910        2794 : SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
    1911        8382 :   unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
    1912        2794 :   Type *Ty1 = VT1.getTypeForEVT(*getContext());
    1913        2794 :   Type *Ty2 = VT2.getTypeForEVT(*getContext());
    1914        2794 :   const DataLayout &DL = getDataLayout();
    1915             :   unsigned Align =
    1916        5588 :       std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
    1917             : 
    1918        2794 :   MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
    1919        2794 :   int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
    1920        8382 :   return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
    1921             : }
    1922             : 
    1923      196200 : SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
    1924             :                                 ISD::CondCode Cond, const SDLoc &dl) {
    1925      196200 :   EVT OpVT = N1.getValueType();
    1926             : 
    1927             :   // These setcc operations always fold.
    1928      196200 :   switch (Cond) {
    1929             :   default: break;
    1930          35 :   case ISD::SETFALSE:
    1931          35 :   case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
    1932          40 :   case ISD::SETTRUE:
    1933          40 :   case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
    1934             : 
    1935             :   case ISD::SETOEQ:
    1936             :   case ISD::SETOGT:
    1937             :   case ISD::SETOGE:
    1938             :   case ISD::SETOLT:
    1939             :   case ISD::SETOLE:
    1940             :   case ISD::SETONE:
    1941             :   case ISD::SETO:
    1942             :   case ISD::SETUO:
    1943             :   case ISD::SETUEQ:
    1944             :   case ISD::SETUNE:
    1945             :     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
    1946             :     break;
    1947             :   }
    1948             : 
    1949             :   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
    1950      119853 :     const APInt &C2 = N2C->getAPIntValue();
    1951             :     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
    1952         566 :       const APInt &C1 = N1C->getAPIntValue();
    1953             : 
    1954         566 :       switch (Cond) {
    1955           0 :       default: llvm_unreachable("Unknown integer setcc!");
    1956         205 :       case ISD::SETEQ:  return getBoolConstant(C1 == C2, dl, VT, OpVT);
    1957         131 :       case ISD::SETNE:  return getBoolConstant(C1 != C2, dl, VT, OpVT);
    1958          21 :       case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
    1959          18 :       case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
    1960          38 :       case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
    1961          16 :       case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
    1962          64 :       case ISD::SETLT:  return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
    1963          56 :       case ISD::SETGT:  return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
    1964           4 :       case ISD::SETLE:  return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
    1965          13 :       case ISD::SETGE:  return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
    1966             :       }
    1967             :     }
    1968             :   }
    1969             :   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) {
    1970             :     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) {
    1971         663 :       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
    1972         221 :       switch (Cond) {
    1973             :       default: break;
    1974           0 :       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
    1975           0 :                           return getUNDEF(VT);
    1976             :                         LLVM_FALLTHROUGH;
    1977             :       case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
    1978          88 :                                                OpVT);
    1979           0 :       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
    1980           0 :                           return getUNDEF(VT);
    1981             :                         LLVM_FALLTHROUGH;
    1982           0 :       case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
    1983           0 :                                                R==APFloat::cmpLessThan, dl, VT,
    1984           0 :                                                OpVT);
    1985           0 :       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
    1986           0 :                           return getUNDEF(VT);
    1987             :                         LLVM_FALLTHROUGH;
    1988             :       case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
    1989          22 :                                                OpVT);
    1990           0 :       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
    1991           0 :                           return getUNDEF(VT);
    1992             :                         LLVM_FALLTHROUGH;
    1993             :       case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
    1994           2 :                                                VT, OpVT);
    1995           0 :       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
    1996           0 :                           return getUNDEF(VT);
    1997             :                         LLVM_FALLTHROUGH;
    1998             :       case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
    1999             :                                                R==APFloat::cmpEqual, dl, VT,
    2000           4 :                                                OpVT);
    2001           0 :       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
    2002           0 :                           return getUNDEF(VT);
    2003             :                         LLVM_FALLTHROUGH;
    2004           5 :       case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
    2005           5 :                                            R==APFloat::cmpEqual, dl, VT, OpVT);
    2006           8 :       case ISD::SETO:   return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
    2007           8 :                                                OpVT);
    2008          31 :       case ISD::SETUO:  return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
    2009          31 :                                                OpVT);
    2010           0 :       case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
    2011           0 :                                                R==APFloat::cmpEqual, dl, VT,
    2012           0 :                                                OpVT);
    2013          16 :       case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
    2014          16 :                                                OpVT);
    2015          12 :       case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
    2016           6 :                                                R==APFloat::cmpLessThan, dl, VT,
    2017           6 :                                                OpVT);
    2018          36 :       case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
    2019             :                                                R==APFloat::cmpUnordered, dl, VT,
    2020          36 :                                                OpVT);
    2021           1 :       case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
    2022           1 :                                                VT, OpVT);
    2023           2 :       case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
    2024           2 :                                                OpVT);
    2025             :       }
    2026             :     } else {
    2027             :       // Ensure that the constant occurs on the RHS.
    2028         231 :       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
    2029             :       MVT CompVT = N1.getValueType().getSimpleVT();
    2030         462 :       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
    2031          95 :         return SDValue();
    2032             : 
    2033         136 :       return getSetCC(dl, VT, N2, N1, SwappedCond);
    2034             :     }
    2035             :   }
    2036             : 
    2037             :   // Could not fold it.
    2038      195107 :   return SDValue();
    2039             : }
    2040             : 
    2041             : /// See if the specified operand can be simplified with the knowledge that only
    2042             : /// the bits specified by Mask are used.
    2043      219846 : SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) {
    2044      219846 :   switch (V.getOpcode()) {
    2045             :   default:
    2046             :     break;
    2047        4824 :   case ISD::Constant: {
    2048             :     const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode());
    2049             :     assert(CV && "Const value should be ConstSDNode.");
    2050        4824 :     const APInt &CVal = CV->getAPIntValue();
    2051        4824 :     APInt NewVal = CVal & Mask;
    2052        4824 :     if (NewVal != CVal)
    2053         460 :       return getConstant(NewVal, SDLoc(V), V.getValueType());
    2054             :     break;
    2055             :   }
    2056        3861 :   case ISD::OR:
    2057             :   case ISD::XOR:
    2058             :     // If the LHS or RHS don't contribute bits to the or, drop them.
    2059        3861 :     if (MaskedValueIsZero(V.getOperand(0), Mask))
    2060         700 :       return V.getOperand(1);
    2061        3161 :     if (MaskedValueIsZero(V.getOperand(1), Mask))
    2062         800 :       return V.getOperand(0);
    2063             :     break;
    2064       41179 :   case ISD::SRL:
    2065             :     // Only look at single-use SRLs.
    2066             :     if (!V.getNode()->hasOneUse())
    2067             :       break;
    2068             :     if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
    2069             :       // See if we can recursively simplify the LHS.
    2070       76302 :       unsigned Amt = RHSC->getZExtValue();
    2071             : 
    2072             :       // Watch out for shift count overflow though.
    2073       38151 :       if (Amt >= Mask.getBitWidth())
    2074             :         break;
    2075             :       APInt NewMask = Mask << Amt;
    2076       38150 :       if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask))
    2077        1171 :         return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
    2078        2342 :                        V.getOperand(1));
    2079             :     }
    2080             :     break;
    2081        7883 :   case ISD::AND: {
    2082             :     // X & -1 -> X (ignoring bits which aren't demanded).
    2083        7883 :     ConstantSDNode *AndVal = isConstOrConstSplat(V.getOperand(1));
    2084       15460 :     if (AndVal && Mask.isSubsetOf(AndVal->getAPIntValue()))
    2085        1347 :       return V.getOperand(0);
    2086             :     break;
    2087             :   }
    2088         613 :   case ISD::ANY_EXTEND: {
    2089         613 :     SDValue Src = V.getOperand(0);
    2090         613 :     unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
    2091             :     // Being conservative here - only peek through if we only demand bits in the
    2092             :     // non-extended source (even though the extended bits are technically undef).
    2093         613 :     if (Mask.getActiveBits() > SrcBitWidth)
    2094             :       break;
    2095         611 :     APInt SrcMask = Mask.trunc(SrcBitWidth);
    2096         611 :     if (SDValue DemandedSrc = GetDemandedBits(Src, SrcMask))
    2097         126 :       return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
    2098             :     break;
    2099             :   }
    2100             :   }
    2101      215535 :   return SDValue();
    2102             : }
    2103             : 
    2104             : /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
    2105             : /// use this predicate to simplify operations downstream.
    2106       61697 : bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
    2107       61697 :   unsigned BitWidth = Op.getScalarValueSizeInBits();
    2108      123394 :   return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
    2109             : }
    2110             : 
    2111             : /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
    2112             : /// this predicate to simplify operations downstream.  Mask is known to be zero
    2113             : /// for bits that V cannot have.
    2114     1336574 : bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
    2115             :                                      unsigned Depth) const {
    2116     1336574 :   KnownBits Known;
    2117     1336574 :   computeKnownBits(Op, Known, Depth);
    2118     1336574 :   return Mask.isSubsetOf(Known.Zero);
    2119             : }
    2120             : 
    2121             : /// Helper function that checks to see if a node is a constant or a
    2122             : /// build vector of splat constants at least within the demanded elts.
    2123       19163 : static ConstantSDNode *isConstOrDemandedConstSplat(SDValue N,
    2124             :                                                    const APInt &DemandedElts) {
    2125             :   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
    2126             :     return CN;
    2127       12462 :   if (N.getOpcode() != ISD::BUILD_VECTOR)
    2128             :     return nullptr;
    2129        9050 :   EVT VT = N.getValueType();
    2130             :   ConstantSDNode *Cst = nullptr;
    2131        4525 :   unsigned NumElts = VT.getVectorNumElements();
    2132             :   assert(DemandedElts.getBitWidth() == NumElts && "Unexpected vector size");
    2133       69553 :   for (unsigned i = 0; i != NumElts; ++i) {
    2134       32569 :     if (!DemandedElts[i])
    2135             :       continue;
    2136             :     ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(i));
    2137       36295 :     if (!C || (Cst && Cst->getAPIntValue() != C->getAPIntValue()) ||
    2138       30156 :         C->getValueType(0) != VT.getScalarType())
    2139             :       return nullptr;
    2140             :     Cst = C;
    2141             :   }
    2142             :   return Cst;
    2143             : }
    2144             : 
    2145             : /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
    2146             : /// is less than the element bit-width of the shift node, return it.
    2147     1538178 : static const APInt *getValidShiftAmountConstant(SDValue V) {
    2148     3076356 :   if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
    2149             :     // Shifting more than the bitwidth is not valid.
    2150     1507236 :     const APInt &ShAmt = SA->getAPIntValue();
    2151     1507236 :     if (ShAmt.ult(V.getScalarValueSizeInBits()))
    2152             :       return &ShAmt;
    2153             :   }
    2154             :   return nullptr;
    2155             : }
    2156             : 
    2157             : /// Determine which bits of Op are known to be either zero or one and return
    2158             : /// them in Known. For vectors, the known bits are those that are shared by
    2159             : /// every vector element.
    2160    12056286 : void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known,
    2161             :                                     unsigned Depth) const {
    2162    24112572 :   EVT VT = Op.getValueType();
    2163             :   APInt DemandedElts = VT.isVector()
    2164             :                            ? APInt::getAllOnesValue(VT.getVectorNumElements())
    2165    12056286 :                            : APInt(1, 1);
    2166    12056286 :   computeKnownBits(Op, Known, DemandedElts, Depth);
    2167    12056286 : }
    2168             : 
    2169             : /// Determine which bits of Op are known to be either zero or one and return
    2170             : /// them in Known. The DemandedElts argument allows us to only collect the known
    2171             : /// bits that are shared by the requested vector elements.
    2172    26193375 : void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known,
    2173             :                                     const APInt &DemandedElts,
    2174             :                                     unsigned Depth) const {
    2175    26193375 :   unsigned BitWidth = Op.getScalarValueSizeInBits();
    2176             : 
    2177    26193375 :   Known = KnownBits(BitWidth);   // Don't know anything.
    2178             : 
    2179             :   if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
    2180             :     // We know all of the bits for a constant!
    2181     9106138 :     Known.One = C->getAPIntValue();
    2182     4553069 :     Known.Zero = ~Known.One;
    2183    10068848 :     return;
    2184             :   }
    2185             :   if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
    2186             :     // We know all of the bits for a constant fp!
    2187        9687 :     Known.One = C->getValueAPF().bitcastToAPInt();
    2188        3229 :     Known.Zero = ~Known.One;
    2189        3229 :     return;
    2190             :   }
    2191             : 
    2192    21637077 :   if (Depth == 6)
    2193             :     return;  // Limit search depth.
    2194             : 
    2195    20677596 :   KnownBits Known2;
    2196    20677596 :   unsigned NumElts = DemandedElts.getBitWidth();
    2197             : 
    2198    20677596 :   if (!DemandedElts)
    2199           0 :     return;  // No demanded elts, better to assume we don't know anything.
    2200             : 
    2201             :   unsigned Opcode = Op.getOpcode();
    2202    20677596 :   switch (Opcode) {
    2203      469193 :   case ISD::BUILD_VECTOR:
    2204             :     // Collect the known bits that are shared by every demanded vector element.
    2205             :     assert(NumElts == Op.getValueType().getVectorNumElements() &&
    2206             :            "Unexpected vector size");
    2207      469193 :     Known.Zero.setAllBits(); Known.One.setAllBits();
    2208     6149936 :     for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
    2209     2733045 :       if (!DemandedElts[i])
    2210     1681175 :         continue;
    2211             : 
    2212     2103740 :       SDValue SrcOp = Op.getOperand(i);
    2213     1051870 :       computeKnownBits(SrcOp, Known2, Depth + 1);
    2214             : 
    2215             :       // BUILD_VECTOR can implicitly truncate sources, we must handle this.
    2216     1051870 :       if (SrcOp.getValueSizeInBits() != BitWidth) {
    2217             :         assert(SrcOp.getValueSizeInBits() > BitWidth &&
    2218             :                "Expected BUILD_VECTOR implicit truncation");
    2219       21892 :         Known2 = Known2.trunc(BitWidth);
    2220             :       }
    2221             : 
    2222             :       // Known bits are the values that are shared by every demanded element.
    2223             :       Known.One &= Known2.One;
    2224             :       Known.Zero &= Known2.Zero;
    2225             : 
    2226             :       // If we don't know any bits, early out.
    2227     1051870 :       if (Known.isUnknown())
    2228             :         break;
    2229      341923 :     }
    2230             :     break;
    2231             :   case ISD::VECTOR_SHUFFLE: {
    2232             :     // Collect the known bits that are shared by every vector element referenced
    2233             :     // by the shuffle.
    2234             :     APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
    2235       81998 :     Known.Zero.setAllBits(); Known.One.setAllBits();
    2236             :     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
    2237             :     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
    2238     1831850 :     for (unsigned i = 0; i != NumElts; ++i) {
    2239      884071 :       if (!DemandedElts[i])
    2240      596357 :         continue;
    2241             : 
    2242      287714 :       int M = SVN->getMaskElt(i);
    2243      287714 :       if (M < 0) {
    2244             :         // For UNDEF elements, we don't know anything about the common state of
    2245             :         // the shuffle result.
    2246             :         Known.resetAll();
    2247        9145 :         DemandedLHS.clearAllBits();
    2248        9145 :         DemandedRHS.clearAllBits();
    2249        9145 :         break;
    2250             :       }
    2251             : 
    2252      278569 :       if ((unsigned)M < NumElts)
    2253      198224 :         DemandedLHS.setBit((unsigned)M % NumElts);
    2254             :       else
    2255       80345 :         DemandedRHS.setBit((unsigned)M % NumElts);
    2256             :     }
    2257             :     // Known bits are the values that are shared by every demanded element.
    2258       81998 :     if (!!DemandedLHS) {
    2259      117618 :       SDValue LHS = Op.getOperand(0);
    2260       58809 :       computeKnownBits(LHS, Known2, DemandedLHS, Depth + 1);
    2261             :       Known.One &= Known2.One;
    2262             :       Known.Zero &= Known2.Zero;
    2263             :     }
    2264             :     // If we don't know any bits, early out.
    2265       81998 :     if (Known.isUnknown())
    2266             :       break;
    2267       16014 :     if (!!DemandedRHS) {
    2268       28570 :       SDValue RHS = Op.getOperand(1);
    2269       14285 :       computeKnownBits(RHS, Known2, DemandedRHS, Depth + 1);
    2270             :       Known.One &= Known2.One;
    2271             :       Known.Zero &= Known2.Zero;
    2272             :     }
    2273             :     break;
    2274             :   }
    2275       36020 :   case ISD::CONCAT_VECTORS: {
    2276             :     // Split DemandedElts and test each of the demanded subvectors.
    2277       36020 :     Known.Zero.setAllBits(); Known.One.setAllBits();
    2278      108060 :     EVT SubVectorVT = Op.getOperand(0).getValueType();
    2279       36020 :     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
    2280             :     unsigned NumSubVectors = Op.getNumOperands();
    2281       99116 :     for (unsigned i = 0; i != NumSubVectors; ++i) {
    2282       61932 :       APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
    2283      123864 :       DemandedSub = DemandedSub.trunc(NumSubVectorElts);
    2284       61932 :       if (!!DemandedSub) {
    2285       72964 :         SDValue Sub = Op.getOperand(i);
    2286       36482 :         computeKnownBits(Sub, Known2, DemandedSub, Depth + 1);
    2287             :         Known.One &= Known2.One;
    2288             :         Known.Zero &= Known2.Zero;
    2289             :       }
    2290             :       // If we don't know any bits, early out.
    2291       61932 :       if (Known.isUnknown())
    2292             :         break;
    2293             :     }
    2294             :     break;
    2295             :   }
    2296       25376 :   case ISD::INSERT_SUBVECTOR: {
    2297             :     // If we know the element index, demand any elements from the subvector and
    2298             :     // the remainder from the src its inserted into, otherwise demand them all.
    2299       25376 :     SDValue Src = Op.getOperand(0);
    2300       25376 :     SDValue Sub = Op.getOperand(1);
    2301             :     ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
    2302       25376 :     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
    2303       76128 :     if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
    2304       25376 :       Known.One.setAllBits();
    2305       25376 :       Known.Zero.setAllBits();
    2306       25376 :       uint64_t Idx = SubIdx->getZExtValue();
    2307       25376 :       APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
    2308       25376 :       if (!!DemandedSubElts) {
    2309       16662 :         computeKnownBits(Sub, Known, DemandedSubElts, Depth + 1);
    2310       16662 :         if (Known.isUnknown())
    2311             :           break; // early-out.
    2312             :       }
    2313        9705 :       APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
    2314       19410 :       APInt DemandedSrcElts = DemandedElts & ~SubMask;
    2315        9705 :       if (!!DemandedSrcElts) {
    2316        8976 :         computeKnownBits(Src, Known2, DemandedSrcElts, Depth + 1);
    2317             :         Known.One &= Known2.One;
    2318             :         Known.Zero &= Known2.Zero;
    2319             :       }
    2320             :     } else {
    2321           0 :       computeKnownBits(Sub, Known, Depth + 1);
    2322           0 :       if (Known.isUnknown())
    2323             :         break; // early-out.
    2324           0 :       computeKnownBits(Src, Known2, Depth + 1);
    2325           0 :       Known.One &= Known2.One;
    2326           0 :       Known.Zero &= Known2.Zero;
    2327             :     }
    2328             :     break;
    2329             :   }
    2330      127576 :   case ISD::EXTRACT_SUBVECTOR: {
    2331             :     // If we know the element index, just demand that subvector elements,
    2332             :     // otherwise demand them all.
    2333      127576 :     SDValue Src = Op.getOperand(0);
    2334             :     ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
    2335      127576 :     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
    2336      382728 :     if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
    2337             :       // Offset the demanded elts by the subvector index.
    2338             :       uint64_t Idx = SubIdx->getZExtValue();
    2339      255152 :       APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx);
    2340      127576 :       computeKnownBits(Src, Known, DemandedSrc, Depth + 1);
    2341             :     } else {
    2342           0 :       computeKnownBits(Src, Known, Depth + 1);
    2343             :     }
    2344             :     break;
    2345             :   }
    2346      828351 :   case ISD::BITCAST: {
    2347      828351 :     SDValue N0 = Op.getOperand(0);
    2348      828351 :     EVT SubVT = N0.getValueType();
    2349             :     unsigned SubBitWidth = SubVT.getScalarSizeInBits();
    2350             : 
    2351             :     // Ignore bitcasts from unsupported types.
    2352      828351 :     if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
    2353             :       break;
    2354             : 
    2355             :     // Fast handling of 'identity' bitcasts.
    2356      828121 :     if (BitWidth == SubBitWidth) {
    2357       29414 :       computeKnownBits(N0, Known, DemandedElts, Depth + 1);
    2358       29414 :       break;
    2359             :     }
    2360             : 
    2361             :     // Support big-endian targets when it becomes useful.
    2362     1597414 :     bool IsLE = getDataLayout().isLittleEndian();
    2363      798707 :     if (!IsLE)
    2364             :       break;
    2365             : 
    2366             :     // Bitcast 'small element' vector to 'large element' scalar/vector.
    2367      789305 :     if ((BitWidth % SubBitWidth) == 0) {
    2368             :       assert(N0.getValueType().isVector() && "Expected bitcast from vector");
    2369             : 
    2370             :       // Collect known bits for the (larger) output by collecting the known
    2371             :       // bits from each set of sub elements and shift these into place.
    2372             :       // We need to separately call computeKnownBits for each set of
    2373             :       // sub elements as the knownbits for each is likely to be different.
    2374      432382 :       unsigned SubScale = BitWidth / SubBitWidth;
    2375      432382 :       APInt SubDemandedElts(NumElts * SubScale, 0);
    2376     2706312 :       for (unsigned i = 0; i != NumElts; ++i)
    2377     1136965 :         if (DemandedElts[i])
    2378      892176 :           SubDemandedElts.setBit(i * SubScale);
    2379             : 
    2380     3572834 :       for (unsigned i = 0; i != SubScale; ++i) {
    2381     3140452 :         computeKnownBits(N0, Known2, SubDemandedElts.shl(i),
    2382             :                          Depth + 1);
    2383     4710678 :         Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * i);
    2384     4710678 :         Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * i);
    2385             :       }
    2386             :     }
    2387             : 
    2388             :     // Bitcast 'large element' scalar/vector to 'small element' vector.
    2389      789305 :     if ((SubBitWidth % BitWidth) == 0) {
    2390             :       assert(Op.getValueType().isVector() && "Expected bitcast to vector");
    2391             : 
    2392             :       // Collect known bits for the (smaller) output by collecting the known
    2393             :       // bits from the overlapping larger input elements and extracting the
    2394             :       // sub sections we actually care about.
    2395      356923 :       unsigned SubScale = SubBitWidth / BitWidth;
    2396      356923 :       APInt SubDemandedElts(NumElts / SubScale, 0);
    2397    16425113 :       for (unsigned i = 0; i != NumElts; ++i)
    2398     8034095 :         if (DemandedElts[i])
    2399     1095259 :           SubDemandedElts.setBit(i / SubScale);
    2400             : 
    2401      356923 :       computeKnownBits(N0, Known2, SubDemandedElts, Depth + 1);
    2402             : 
    2403      356923 :       Known.Zero.setAllBits(); Known.One.setAllBits();
    2404     6438761 :       for (unsigned i = 0; i != NumElts; ++i)
    2405     3379969 :         if (DemandedElts[i]) {
    2406      453501 :           unsigned Offset = (i % SubScale) * BitWidth;
    2407     1360503 :           Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
    2408     1360503 :           Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
    2409             :           // If we don't know any bits, early out.
    2410      453501 :           if (Known.isUnknown())
    2411             :             break;
    2412             :         }
    2413             :     }
    2414             :     break;
    2415             :   }
    2416      664236 :   case ISD::AND:
    2417             :     // If either the LHS or the RHS are Zero, the result is zero.
    2418     1328472 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2419     1328472 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2420             : 
    2421             :     // Output known-1 bits are only known if set in both the LHS & RHS.
    2422      664236 :     Known.One &= Known2.One;
    2423             :     // Output known-0 are known to be clear if zero in either the LHS | RHS.
    2424      664236 :     Known.Zero |= Known2.Zero;
    2425             :     break;
    2426      175002 :   case ISD::OR:
    2427      350004 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2428      350004 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2429             : 
    2430             :     // Output known-0 bits are only known if clear in both the LHS & RHS.
    2431      175002 :     Known.Zero &= Known2.Zero;
    2432             :     // Output known-1 are known to be set if set in either the LHS | RHS.
    2433      175002 :     Known.One |= Known2.One;
    2434             :     break;
    2435      163753 :   case ISD::XOR: {
    2436      327506 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2437      327506 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2438             : 
    2439             :     // Output known-0 bits are known if clear or set in both the LHS & RHS.
    2440      982518 :     APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
    2441             :     // Output known-1 are known to be set if set in only one of the LHS, RHS.
    2442      655012 :     Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
    2443      163753 :     Known.Zero = KnownZeroOut;
    2444             :     break;
    2445             :   }
    2446       92064 :   case ISD::MUL: {
    2447      184128 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2448      184128 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2449             : 
    2450             :     // If low bits are zero in either operand, output low known-0 bits.
    2451             :     // Also compute a conservative estimate for high known-0 bits.
    2452             :     // More trickiness is possible, but this is sufficient for the
    2453             :     // interesting case of alignment computation.
    2454       92064 :     unsigned TrailZ = Known.countMinTrailingZeros() +
    2455       92064 :                       Known2.countMinTrailingZeros();
    2456      276192 :     unsigned LeadZ =  std::max(Known.countMinLeadingZeros() +
    2457             :                                Known2.countMinLeadingZeros(),
    2458       92064 :                                BitWidth) - BitWidth;
    2459             : 
    2460             :     Known.resetAll();
    2461       92064 :     Known.Zero.setLowBits(std::min(TrailZ, BitWidth));
    2462       92064 :     Known.Zero.setHighBits(std::min(LeadZ, BitWidth));
    2463             :     break;
    2464             :   }
    2465        1118 :   case ISD::UDIV: {
    2466             :     // For the purposes of computing leading zeros we can conservatively
    2467             :     // treat a udiv as a logical right shift by the power of 2 known to
    2468             :     // be less than the denominator.
    2469        2236 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2470             :     unsigned LeadZ = Known2.countMinLeadingZeros();
    2471             : 
    2472        2236 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2473             :     unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
    2474        1118 :     if (RHSMaxLeadingZeros != BitWidth)
    2475        1154 :       LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
    2476             : 
    2477        1118 :     Known.Zero.setHighBits(LeadZ);
    2478             :     break;
    2479             :   }
    2480      141146 :   case ISD::SELECT:
    2481             :   case ISD::VSELECT:
    2482      282292 :     computeKnownBits(Op.getOperand(2), Known, DemandedElts, Depth+1);
    2483             :     // If we don't know any bits, early out.
    2484      141146 :     if (Known.isUnknown())
    2485             :       break;
    2486      215180 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth+1);
    2487             : 
    2488             :     // Only known if known in both the LHS and RHS.
    2489      107590 :     Known.One &= Known2.One;
    2490      107590 :     Known.Zero &= Known2.Zero;
    2491             :     break;
    2492       35825 :   case ISD::SELECT_CC:
    2493       71650 :     computeKnownBits(Op.getOperand(3), Known, DemandedElts, Depth+1);
    2494             :     // If we don't know any bits, early out.
    2495       35825 :     if (Known.isUnknown())
    2496             :       break;
    2497       13124 :     computeKnownBits(Op.getOperand(2), Known2, DemandedElts, Depth+1);
    2498             : 
    2499             :     // Only known if known in both the LHS and RHS.
    2500        6562 :     Known.One &= Known2.One;
    2501        6562 :     Known.Zero &= Known2.Zero;
    2502             :     break;
    2503        1982 :   case ISD::SMULO:
    2504             :   case ISD::UMULO:
    2505             :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
    2506        1982 :     if (Op.getResNo() != 1)
    2507             :       break;
    2508             :     // The boolean result conforms to getBooleanContents.
    2509             :     // If we know the result of a setcc has the top bits zero, use this info.
    2510             :     // We know that we have an integer-based boolean since these operations
    2511             :     // are only available for integer.
    2512        4143 :     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
    2513        1863 :             TargetLowering::ZeroOrOneBooleanContent &&
    2514             :         BitWidth > 1)
    2515        1446 :       Known.Zero.setBitsFrom(1);
    2516             :     break;
    2517      582210 :   case ISD::SETCC:
    2518             :     // If we know the result of a setcc has the top bits zero, use this info.
    2519     1746630 :     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    2520      582210 :             TargetLowering::ZeroOrOneBooleanContent &&
    2521             :         BitWidth > 1)
    2522      311699 :       Known.Zero.setBitsFrom(1);
    2523             :     break;
    2524      950430 :   case ISD::SHL:
    2525      950430 :     if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
    2526     1864518 :       computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2527      932259 :       unsigned Shift = ShAmt->getZExtValue();
    2528      932259 :       Known.Zero <<= Shift;
    2529      932259 :       Known.One <<= Shift;
    2530             :       // Low bits are known zero.
    2531             :       Known.Zero.setLowBits(Shift);
    2532             :     }
    2533             :     break;
    2534      546734 :   case ISD::SRL:
    2535      546734 :     if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
    2536     1070232 :       computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2537      535116 :       unsigned Shift = ShAmt->getZExtValue();
    2538      535116 :       Known.Zero.lshrInPlace(Shift);
    2539      535116 :       Known.One.lshrInPlace(Shift);
    2540             :       // High bits are known zero.
    2541             :       Known.Zero.setHighBits(Shift);
    2542       11618 :     } else if (auto *BV = dyn_cast<BuildVectorSDNode>(Op.getOperand(1))) {
    2543             :       // If the shift amount is a vector of constants see if we can bound
    2544             :       // the number of upper zero bits.
    2545         193 :       unsigned ShiftAmountMin = BitWidth;
    2546        5054 :       for (unsigned i = 0; i != BV->getNumOperands(); ++i) {
    2547        1620 :         if (auto *C = dyn_cast<ConstantSDNode>(BV->getOperand(i))) {
    2548        1586 :           const APInt &ShAmt = C->getAPIntValue();
    2549        3142 :           if (ShAmt.ult(BitWidth)) {
    2550        1556 :             ShiftAmountMin = std::min<unsigned>(ShiftAmountMin,
    2551        3112 :                                                 ShAmt.getZExtValue());
    2552             :             continue;
    2553             :           }
    2554             :         }
    2555             :         // Don't know anything.
    2556          64 :         ShiftAmountMin = 0;
    2557          64 :         break;
    2558             :       }
    2559             : 
    2560         193 :       Known.Zero.setHighBits(ShiftAmountMin);
    2561             :     }
    2562             :     break;
    2563       41014 :   case ISD::SRA:
    2564       41014 :     if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
    2565       79654 :       computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2566       39827 :       unsigned Shift = ShAmt->getZExtValue();
    2567             :       // Sign extend known zero/one bit (else is unknown).
    2568       39827 :       Known.Zero.ashrInPlace(Shift);
    2569       39827 :       Known.One.ashrInPlace(Shift);
    2570             :     }
    2571             :     break;
    2572       28984 :   case ISD::SIGN_EXTEND_INREG: {
    2573       28984 :     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
    2574             :     unsigned EBits = EVT.getScalarSizeInBits();
    2575             : 
    2576             :     // Sign extension.  Compute the demanded bits in the result that are not
    2577             :     // present in the input.
    2578       28984 :     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
    2579             : 
    2580             :     APInt InSignMask = APInt::getSignMask(EBits);
    2581       28984 :     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
    2582             : 
    2583             :     // If the sign extended bits are demanded, we know that the sign
    2584             :     // bit is demanded.
    2585       57968 :     InSignMask = InSignMask.zext(BitWidth);
    2586       28984 :     if (NewBits.getBoolValue())
    2587             :       InputDemandedBits |= InSignMask;
    2588             : 
    2589       57968 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2590       28984 :     Known.One &= InputDemandedBits;
    2591       28984 :     Known.Zero &= InputDemandedBits;
    2592             : 
    2593             :     // If the sign bit of the input is known set or clear, then we know the
    2594             :     // top bits of the result.
    2595       28984 :     if (Known.Zero.intersects(InSignMask)) {        // Input sign bit known clear
    2596             :       Known.Zero |= NewBits;
    2597           4 :       Known.One  &= ~NewBits;
    2598       28982 :     } else if (Known.One.intersects(InSignMask)) {  // Input sign bit known set
    2599             :       Known.One  |= NewBits;
    2600           4 :       Known.Zero &= ~NewBits;
    2601             :     } else {                              // Input sign bit unknown
    2602       57960 :       Known.Zero &= ~NewBits;
    2603       57960 :       Known.One  &= ~NewBits;
    2604             :     }
    2605             :     break;
    2606             :   }
    2607         612 :   case ISD::CTTZ:
    2608             :   case ISD::CTTZ_ZERO_UNDEF: {
    2609        1224 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2610             :     // If we have a known 1, its position is our upper bound.
    2611             :     unsigned PossibleTZ = Known2.countMaxTrailingZeros();
    2612         612 :     unsigned LowBits = Log2_32(PossibleTZ) + 1;
    2613         612 :     Known.Zero.setBitsFrom(LowBits);
    2614             :     break;
    2615             :   }
    2616        2445 :   case ISD::CTLZ:
    2617             :   case ISD::CTLZ_ZERO_UNDEF: {
    2618        4890 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2619             :     // If we have a known 1, its position is our upper bound.
    2620             :     unsigned PossibleLZ = Known2.countMaxLeadingZeros();
    2621        2445 :     unsigned LowBits = Log2_32(PossibleLZ) + 1;
    2622        2445 :     Known.Zero.setBitsFrom(LowBits);
    2623             :     break;
    2624             :   }
    2625        1650 :   case ISD::CTPOP: {
    2626        3300 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2627             :     // If we know some of the bits are zero, they can't be one.
    2628             :     unsigned PossibleOnes = Known2.countMaxPopulation();
    2629        3300 :     Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
    2630             :     break;
    2631             :   }
    2632             :   case ISD::LOAD: {
    2633             :     LoadSDNode *LD = cast<LoadSDNode>(Op);
    2634             :     // If this is a ZEXTLoad and we are looking at the loaded value.
    2635       64128 :     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
    2636       64050 :       EVT VT = LD->getMemoryVT();
    2637             :       unsigned MemBits = VT.getScalarSizeInBits();
    2638       64050 :       Known.Zero.setBitsFrom(MemBits);
    2639     4310696 :     } else if (const MDNode *Ranges = LD->getRanges()) {
    2640        1851 :       if (LD->getExtensionType() == ISD::NON_EXTLOAD)
    2641        1845 :         computeKnownBitsFromRangeMetadata(*Ranges, Known);
    2642             :     }
    2643             :     break;
    2644             :   }
    2645        5014 :   case ISD::ZERO_EXTEND_VECTOR_INREG: {
    2646       10028 :     EVT InVT = Op.getOperand(0).getValueType();
    2647        5014 :     APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
    2648       10028 :     computeKnownBits(Op.getOperand(0), Known, InDemandedElts, Depth + 1);
    2649        5014 :     Known = Known.zext(BitWidth);
    2650        5014 :     Known.Zero.setBitsFrom(InVT.getScalarSizeInBits());
    2651             :     break;
    2652             :   }
    2653      593305 :   case ISD::ZERO_EXTEND: {
    2654     1186610 :     EVT InVT = Op.getOperand(0).getValueType();
    2655     1186610 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2656      593305 :     Known = Known.zext(BitWidth);
    2657      593305 :     Known.Zero.setBitsFrom(InVT.getScalarSizeInBits());
    2658             :     break;
    2659             :   }
    2660             :   // TODO ISD::SIGN_EXTEND_VECTOR_INREG
    2661       47650 :   case ISD::SIGN_EXTEND: {
    2662       95300 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2663             :     // If the sign bit is known to be zero or one, then sext will extend
    2664             :     // it to the top bits, else it will just zext.
    2665       47650 :     Known = Known.sext(BitWidth);
    2666       47650 :     break;
    2667             :   }
    2668      191996 :   case ISD::ANY_EXTEND: {
    2669      383992 :     computeKnownBits(Op.getOperand(0), Known, Depth+1);
    2670      191996 :     Known = Known.zext(BitWidth);
    2671      191996 :     break;
    2672             :   }
    2673      386907 :   case ISD::TRUNCATE: {
    2674      773814 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2675      386907 :     Known = Known.trunc(BitWidth);
    2676      386907 :     break;
    2677             :   }
    2678      271747 :   case ISD::AssertZext: {
    2679      271747 :     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
    2680      271747 :     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
    2681      543494 :     computeKnownBits(Op.getOperand(0), Known, Depth+1);
    2682      815241 :     Known.Zero |= (~InMask);
    2683      815241 :     Known.One  &= (~Known.Zero);
    2684             :     break;
    2685             :   }
    2686          26 :   case ISD::FGETSIGN:
    2687             :     // All bits are zero except the low bit.
    2688          26 :     Known.Zero.setBitsFrom(1);
    2689             :     break;
    2690        4678 :   case ISD::USUBO:
    2691             :   case ISD::SSUBO:
    2692        4678 :     if (Op.getResNo() == 1) {
    2693             :       // If we know the result of a setcc has the top bits zero, use this info.
    2694       13395 :       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    2695        4465 :               TargetLowering::ZeroOrOneBooleanContent &&
    2696             :           BitWidth > 1)
    2697         155 :         Known.Zero.setBitsFrom(1);
    2698             :       break;
    2699             :     }
    2700             :     LLVM_FALLTHROUGH;
    2701             :   case ISD::SUB:
    2702             :   case ISD::SUBC: {
    2703       83609 :     if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0))) {
    2704             :       // We know that the top bits of C-X are clear if X contains less bits
    2705             :       // than C (i.e. no wrap-around can happen).  For example, 20-X is
    2706             :       // positive if we can prove that X is >= 0 and < 16.
    2707       29734 :       if (CLHS->getAPIntValue().isNonNegative()) {
    2708       43587 :         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
    2709             :         // NLZ can't be BitWidth with no sign bit
    2710       14529 :         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
    2711       29058 :         computeKnownBits(Op.getOperand(1), Known2, DemandedElts,
    2712             :                          Depth + 1);
    2713             : 
    2714             :         // If all of the MaskV bits are known to be zero, then we know the
    2715             :         // output top bits are zero, because we now know that the output is
    2716             :         // from [0-C].
    2717       29058 :         if ((Known2.Zero & MaskV) == MaskV) {
    2718        4548 :           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
    2719             :           // Top bits known zero.
    2720        2274 :           Known.Zero.setHighBits(NLZ2);
    2721             :         }
    2722             :       }
    2723             :     }
    2724             : 
    2725             :     // If low bits are know to be zero in both operands, then we know they are
    2726             :     // going to be 0 in the result. Both addition and complement operations
    2727             :     // preserve the low zero bits.
    2728      167218 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2729       83609 :     unsigned KnownZeroLow = Known2.countMinTrailingZeros();
    2730       83609 :     if (KnownZeroLow == 0)
    2731             :       break;
    2732             : 
    2733       29308 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2734       29308 :     KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
    2735       14654 :     Known.Zero.setLowBits(KnownZeroLow);
    2736             :     break;
    2737             :   }
    2738       19222 :   case ISD::UADDO:
    2739             :   case ISD::SADDO:
    2740             :   case ISD::ADDCARRY:
    2741       19222 :     if (Op.getResNo() == 1) {
    2742             :       // If we know the result of a setcc has the top bits zero, use this info.
    2743        4983 :       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    2744        1661 :               TargetLowering::ZeroOrOneBooleanContent &&
    2745             :           BitWidth > 1)
    2746        1371 :         Known.Zero.setBitsFrom(1);
    2747             :       break;
    2748             :     }
    2749             :     LLVM_FALLTHROUGH;
    2750             :   case ISD::ADD:
    2751             :   case ISD::ADDC:
    2752             :   case ISD::ADDE: {
    2753             :     // Output known-0 bits are known if clear or set in both the low clear bits
    2754             :     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
    2755             :     // low 3 bits clear.
    2756             :     // Output known-0 bits are also known if the top bits of each input are
    2757             :     // known to be clear. For example, if one input has the top 10 bits clear
    2758             :     // and the other has the top 8 bits clear, we know the top 7 bits of the
    2759             :     // output must be clear.
    2760     5909438 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2761     2954719 :     unsigned KnownZeroHigh = Known2.countMinLeadingZeros();
    2762     2954719 :     unsigned KnownZeroLow = Known2.countMinTrailingZeros();
    2763             : 
    2764     5909438 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts,
    2765             :                      Depth + 1);
    2766     5909438 :     KnownZeroHigh = std::min(KnownZeroHigh, Known2.countMinLeadingZeros());
    2767     5909438 :     KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
    2768             : 
    2769     2954719 :     if (Opcode == ISD::ADDE || Opcode == ISD::ADDCARRY) {
    2770             :       // With ADDE and ADDCARRY, a carry bit may be added in, so we can only
    2771             :       // use this information if we know (at least) that the low two bits are
    2772             :       // clear. We then return to the caller that the low bit is unknown but
    2773             :       // that other bits are known zero.
    2774       12603 :       if (KnownZeroLow >= 2)
    2775         132 :         Known.Zero.setBits(1, KnownZeroLow);
    2776             :       break;
    2777             :     }
    2778             : 
    2779     2942116 :     Known.Zero.setLowBits(KnownZeroLow);
    2780     2942116 :     if (KnownZeroHigh > 1)
    2781       35471 :       Known.Zero.setHighBits(KnownZeroHigh - 1);
    2782             :     break;
    2783             :   }
    2784        1264 :   case ISD::SREM:
    2785        1264 :     if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
    2786         720 :       const APInt &RA = Rem->getAPIntValue().abs();
    2787         360 :       if (RA.isPowerOf2()) {
    2788          22 :         APInt LowBits = RA - 1;
    2789          44 :         computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2790             : 
    2791             :         // The low bits of the first operand are unchanged by the srem.
    2792          44 :         Known.Zero = Known2.Zero & LowBits;
    2793          44 :         Known.One = Known2.One & LowBits;
    2794             : 
    2795             :         // If the first operand is non-negative or has all low bits zero, then
    2796             :         // the upper bits are all zero.
    2797         110 :         if (Known2.Zero[BitWidth-1] || ((Known2.Zero & LowBits) == LowBits))
    2798           4 :           Known.Zero |= ~LowBits;
    2799             : 
    2800             :         // If the first operand is negative and not all low bits are zero, then
    2801             :         // the upper bits are all one.
    2802          88 :         if (Known2.One[BitWidth-1] && ((Known2.One & LowBits) != 0))
    2803           0 :           Known.One |= ~LowBits;
    2804             :         assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?");
    2805             :       }
    2806             :     }
    2807             :     break;
    2808        1508 :   case ISD::UREM: {
    2809        1508 :     if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
    2810         775 :       const APInt &RA = Rem->getAPIntValue();
    2811         775 :       if (RA.isPowerOf2()) {
    2812           8 :         APInt LowBits = (RA - 1);
    2813          16 :         computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2814             : 
    2815             :         // The upper bits are all zero, the lower ones are unchanged.
    2816          16 :         Known.Zero = Known2.Zero | ~LowBits;
    2817           8 :         Known.One = Known2.One & LowBits;
    2818             :         break;
    2819             :       }
    2820             :     }
    2821             : 
    2822             :     // Since the result is less than or equal to either operand, any leading
    2823             :     // zero bits in either operand must also exist in the result.
    2824        3000 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2825        3000 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2826             : 
    2827             :     uint32_t Leaders =
    2828        4500 :         std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
    2829             :     Known.resetAll();
    2830             :     Known.Zero.setHighBits(Leaders);
    2831             :     break;
    2832             :   }
    2833         128 :   case ISD::EXTRACT_ELEMENT: {
    2834         256 :     computeKnownBits(Op.getOperand(0), Known, Depth+1);
    2835         256 :     const unsigned Index = Op.getConstantOperandVal(1);
    2836         128 :     const unsigned BitWidth = Op.getValueSizeInBits();
    2837             : 
    2838             :     // Remove low part of known bits mask
    2839         256 :     Known.Zero = Known.Zero.getHiBits(Known.Zero.getBitWidth() - Index * BitWidth);
    2840         256 :     Known.One = Known.One.getHiBits(Known.One.getBitWidth() - Index * BitWidth);
    2841             : 
    2842             :     // Remove high part of known bit mask
    2843         128 :     Known = Known.trunc(BitWidth);
    2844         128 :     break;
    2845             :   }
    2846      381607 :   case ISD::EXTRACT_VECTOR_ELT: {
    2847      381607 :     SDValue InVec = Op.getOperand(0);
    2848      381607 :     SDValue EltNo = Op.getOperand(1);
    2849      381607 :     EVT VecVT = InVec.getValueType();
    2850      381607 :     const unsigned BitWidth = Op.getValueSizeInBits();
    2851             :     const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
    2852      381607 :     const unsigned NumSrcElts = VecVT.getVectorNumElements();
    2853             :     // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
    2854             :     // anything about the extended bits.
    2855      381607 :     if (BitWidth > EltBitWidth)
    2856       11810 :       Known = Known.trunc(EltBitWidth);
    2857             :     ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
    2858      761588 :     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) {
    2859             :       // If we know the element index, just demand that vector element.
    2860      380794 :       unsigned Idx = ConstEltNo->getZExtValue();
    2861      380794 :       APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
    2862      380794 :       computeKnownBits(InVec, Known, DemandedElt, Depth + 1);
    2863             :     } else {
    2864             :       // Unknown element index, so ignore DemandedElts and demand them all.
    2865         813 :       computeKnownBits(InVec, Known, Depth + 1);
    2866             :     }
    2867      381607 :     if (BitWidth > EltBitWidth)
    2868       11810 :       Known = Known.zext(BitWidth);
    2869             :     break;
    2870             :   }
    2871        5337 :   case ISD::INSERT_VECTOR_ELT: {
    2872        5337 :     SDValue InVec = Op.getOperand(0);
    2873        5337 :     SDValue InVal = Op.getOperand(1);
    2874        5337 :     SDValue EltNo = Op.getOperand(2);
    2875             : 
    2876             :     ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
    2877       10662 :     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
    2878             :       // If we know the element index, split the demand between the
    2879             :       // source vector and the inserted element.
    2880       15993 :       Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth);
    2881       10662 :       unsigned EltIdx = CEltNo->getZExtValue();
    2882             : 
    2883             :       // If we demand the inserted element then add its common known bits.
    2884        5331 :       if (DemandedElts[EltIdx]) {
    2885        4229 :         computeKnownBits(InVal, Known2, Depth + 1);
    2886        8458 :         Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
    2887        8458 :         Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
    2888             :       }
    2889             : 
    2890             :       // If we demand the source vector then add its common known bits, ensuring
    2891             :       // that we don't demand the inserted element.
    2892       15993 :       APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
    2893        5331 :       if (!!VectorElts) {
    2894        3872 :         computeKnownBits(InVec, Known2, VectorElts, Depth + 1);
    2895             :         Known.One &= Known2.One;
    2896             :         Known.Zero &= Known2.Zero;
    2897             :       }
    2898             :     } else {
    2899             :       // Unknown element index, so ignore DemandedElts and demand them all.
    2900           6 :       computeKnownBits(InVec, Known, Depth + 1);
    2901           6 :       computeKnownBits(InVal, Known2, Depth + 1);
    2902          12 :       Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
    2903          12 :       Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
    2904             :     }
    2905             :     break;
    2906             :   }
    2907         133 :   case ISD::BITREVERSE: {
    2908         266 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2909         266 :     Known.Zero = Known2.Zero.reverseBits();
    2910         266 :     Known.One = Known2.One.reverseBits();
    2911         133 :     break;
    2912             :   }
    2913        1862 :   case ISD::BSWAP: {
    2914        3724 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2915        3724 :     Known.Zero = Known2.Zero.byteSwap();
    2916        3724 :     Known.One = Known2.One.byteSwap();
    2917        1862 :     break;
    2918             :   }
    2919        1790 :   case ISD::ABS: {
    2920        3580 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2921             : 
    2922             :     // If the source's MSB is zero then we know the rest of the bits already.
    2923        1790 :     if (Known2.isNonNegative()) {
    2924           0 :       Known.Zero = Known2.Zero;
    2925           0 :       Known.One = Known2.One;
    2926           0 :       break;
    2927             :     }
    2928             : 
    2929             :     // We only know that the absolute values's MSB will be zero iff there is
    2930             :     // a set bit that isn't the sign bit (otherwise it could be INT_MIN).
    2931        1790 :     Known2.One.clearSignBit();
    2932        1790 :     if (Known2.One.getBoolValue()) {
    2933           2 :       Known.Zero = APInt::getSignMask(BitWidth);
    2934           2 :       break;
    2935             :     }
    2936             :     break;
    2937             :   }
    2938        2506 :   case ISD::UMIN: {
    2939        5012 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2940        5012 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2941             : 
    2942             :     // UMIN - we know that the result will have the maximum of the
    2943             :     // known zero leading bits of the inputs.
    2944        2506 :     unsigned LeadZero = Known.countMinLeadingZeros();
    2945        5012 :     LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros());
    2946             : 
    2947             :     Known.Zero &= Known2.Zero;
    2948        2506 :     Known.One &= Known2.One;
    2949             :     Known.Zero.setHighBits(LeadZero);
    2950             :     break;
    2951             :   }
    2952        2173 :   case ISD::UMAX: {
    2953        4346 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts,
    2954             :                      Depth + 1);
    2955        4346 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2956             : 
    2957             :     // UMAX - we know that the result will have the maximum of the
    2958             :     // known one leading bits of the inputs.
    2959        2173 :     unsigned LeadOne = Known.countMinLeadingOnes();
    2960        4346 :     LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes());
    2961             : 
    2962        2173 :     Known.Zero &= Known2.Zero;
    2963             :     Known.One &= Known2.One;
    2964             :     Known.One.setHighBits(LeadOne);
    2965             :     break;
    2966             :   }
    2967        9115 :   case ISD::SMIN:
    2968             :   case ISD::SMAX: {
    2969             :     // If we have a clamp pattern, we know that the number of sign bits will be
    2970             :     // the minimum of the clamp min/max range.
    2971             :     bool IsMax = (Opcode == ISD::SMAX);
    2972             :     ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
    2973        9115 :     if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)))
    2974        5457 :       if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
    2975         923 :         CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1),
    2976             :                                               DemandedElts);
    2977        9115 :     if (CstLow && CstHigh) {
    2978         917 :       if (!IsMax)
    2979             :         std::swap(CstLow, CstHigh);
    2980             : 
    2981         917 :       const APInt &ValueLow = CstLow->getAPIntValue();
    2982         917 :       const APInt &ValueHigh = CstHigh->getAPIntValue();
    2983         917 :       if (ValueLow.sle(ValueHigh)) {
    2984         917 :         unsigned LowSignBits = ValueLow.getNumSignBits();
    2985         917 :         unsigned HighSignBits = ValueHigh.getNumSignBits();
    2986         917 :         unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
    2987        2136 :         if (ValueLow.isNegative() && ValueHigh.isNegative()) {
    2988           0 :           Known.One.setHighBits(MinSignBits);
    2989         615 :           break;
    2990             :         }
    2991        1532 :         if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
    2992         615 :           Known.Zero.setHighBits(MinSignBits);
    2993             :           break;
    2994             :         }
    2995             :       }
    2996             :     }
    2997             : 
    2998             :     // Fallback - just get the shared known bits of the operands.
    2999       17000 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    3000        8500 :     if (Known.isUnknown()) break; // Early-out
    3001           0 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    3002           0 :     Known.Zero &= Known2.Zero;
    3003           0 :     Known.One &= Known2.One;
    3004             :     break;
    3005             :   }
    3006      921808 :   case ISD::FrameIndex:
    3007             :   case ISD::TargetFrameIndex:
    3008      921808 :     TLI->computeKnownBitsForFrameIndex(Op, Known, DemandedElts, *this, Depth);
    3009      921808 :     break;
    3010             : 
    3011     7498326 :   default:
    3012     7498326 :     if (Opcode < ISD::BUILTIN_OP_END)
    3013             :       break;
    3014             :     LLVM_FALLTHROUGH;
    3015             :   case ISD::INTRINSIC_WO_CHAIN:
    3016             :   case ISD::INTRINSIC_W_CHAIN:
    3017             :   case ISD::INTRINSIC_VOID:
    3018             :     // Allow the target to implement this method for its nodes.
    3019     3565672 :     TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
    3020     3565672 :     break;
    3021             :   }
    3022             : 
    3023             :   assert(!Known.hasConflict() && "Bits known to be one AND zero?");
    3024             : }
    3025             : 
    3026       79766 : SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
    3027             :                                                              SDValue N1) const {
    3028             :   // X + 0 never overflow
    3029       79766 :   if (isNullConstant(N1))
    3030             :     return OFK_Never;
    3031             : 
    3032       79766 :   KnownBits N1Known;
    3033       79766 :   computeKnownBits(N1, N1Known);
    3034       79766 :   if (N1Known.Zero.getBoolValue()) {
    3035       75445 :     KnownBits N0Known;
    3036       75472 :     computeKnownBits(N0, N0Known);
    3037             : 
    3038             :     bool overflow;
    3039      452832 :     (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
    3040       75472 :     if (!overflow)
    3041          27 :       return OFK_Never;
    3042             :   }
    3043             : 
    3044             :   // mulhi + 1 never overflow
    3045      239435 :   if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
    3046      478434 :       (~N1Known.Zero & 0x01) == ~N1Known.Zero)
    3047             :     return OFK_Never;
    3048             : 
    3049      159042 :   if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
    3050         228 :     KnownBits N0Known;
    3051         228 :     computeKnownBits(N0, N0Known);
    3052             : 
    3053        1140 :     if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
    3054           0 :       return OFK_Never;
    3055             :   }
    3056             : 
    3057             :   return OFK_Sometime;
    3058             : }
    3059             : 
    3060       37260 : bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
    3061       74520 :   EVT OpVT = Val.getValueType();
    3062             :   unsigned BitWidth = OpVT.getScalarSizeInBits();
    3063             : 
    3064             :   // Is the constant a known power of 2?
    3065             :   if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
    3066      100947 :     return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
    3067             : 
    3068             :   // A left-shift of a constant one will have exactly one bit set because
    3069             :   // shifting the bit off the end is undefined.
    3070        3611 :   if (Val.getOpcode() == ISD::SHL) {
    3071          47 :     auto *C = isConstOrConstSplat(Val.getOperand(0));
    3072          91 :     if (C && C->getAPIntValue() == 1)
    3073             :       return true;
    3074             :   }
    3075             : 
    3076             :   // Similarly, a logical right-shift of a constant sign-bit will have exactly
    3077             :   // one bit set.
    3078        3570 :   if (Val.getOpcode() == ISD::SRL) {
    3079          25 :     auto *C = isConstOrConstSplat(Val.getOperand(0));
    3080          30 :     if (C && C->getAPIntValue().isSignMask())
    3081             :       return true;
    3082             :   }
    3083             : 
    3084             :   // Are all operands of a build vector constant powers of two?
    3085        3565 :   if (Val.getOpcode() == ISD::BUILD_VECTOR)
    3086       12915 :     if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
    3087             :           if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
    3088       16299 :             return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
    3089             :           return false;
    3090             :         }))
    3091             :       return true;
    3092             : 
    3093             :   // More could be done here, though the above checks are enough
    3094             :   // to handle some common cases.
    3095             : 
    3096             :   // Fall back to computeKnownBits to catch other known cases.
    3097        3212 :   KnownBits Known;
    3098        3212 :   computeKnownBits(Val, Known);
    3099        3345 :   return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
    3100             : }
    3101             : 
    3102      321676 : unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
    3103      643352 :   EVT VT = Op.getValueType();
    3104             :   APInt DemandedElts = VT.isVector()
    3105             :                            ? APInt::getAllOnesValue(VT.getVectorNumElements())
    3106      321676 :                            : APInt(1, 1);
    3107      643352 :   return ComputeNumSignBits(Op, DemandedElts, Depth);
    3108             : }
    3109             : 
    3110      392803 : unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
    3111             :                                           unsigned Depth) const {
    3112      785606 :   EVT VT = Op.getValueType();
    3113             :   assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
    3114      392803 :   unsigned VTBits = VT.getScalarSizeInBits();
    3115      392803 :   unsigned NumElts = DemandedElts.getBitWidth();
    3116             :   unsigned Tmp, Tmp2;
    3117      392803 :   unsigned FirstAnswer = 1;
    3118             : 
    3119             :   if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
    3120       26256 :     const APInt &Val = C->getAPIntValue();
    3121       26256 :     return Val.getNumSignBits();
    3122             :   }
    3123             : 
    3124      366547 :   if (Depth == 6)
    3125             :     return 1;  // Limit search depth.
    3126             : 
    3127      366001 :   if (!DemandedElts)
    3128             :     return 1;  // No demanded elts, better to assume we don't know anything.
    3129             : 
    3130             :   unsigned Opcode = Op.getOpcode();
    3131      366001 :   switch (Opcode) {
    3132      271849 :   default: break;
    3133        3854 :   case ISD::AssertSext:
    3134        3854 :     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
    3135       98006 :     return VTBits-Tmp+1;
    3136        7109 :   case ISD::AssertZext:
    3137        7109 :     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
    3138        7109 :     return VTBits-Tmp;
    3139             : 
    3140        4368 :   case ISD::BUILD_VECTOR:
    3141        4368 :     Tmp = VTBits;
    3142       34040 :     for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
    3143       14836 :       if (!DemandedElts[i])
    3144        3985 :         continue;
    3145             : 
    3146       21702 :       SDValue SrcOp = Op.getOperand(i);
    3147       21702 :       Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1);
    3148             : 
    3149             :       // BUILD_VECTOR can implicitly truncate sources, we must handle this.
    3150       10851 :       if (SrcOp.getValueSizeInBits() != VTBits) {
    3151             :         assert(SrcOp.getValueSizeInBits() > VTBits &&
    3152             :                "Expected BUILD_VECTOR implicit truncation");
    3153          50 :         unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
    3154          50 :         Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
    3155             :       }
    3156       10851 :       Tmp = std::min(Tmp, Tmp2);
    3157        4368 :     }
    3158        4368 :     return Tmp;
    3159             : 
    3160             :   case ISD::VECTOR_SHUFFLE: {
    3161             :     // Collect the minimum number of sign bits that are shared by every vector
    3162             :     // element referenced by the shuffle.
    3163             :     APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
    3164             :     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
    3165             :     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
    3166       14665 :     for (unsigned i = 0; i != NumElts; ++i) {
    3167        6899 :       int M = SVN->getMaskElt(i);
    3168        6899 :       if (!DemandedElts[i])
    3169        1726 :         continue;
    3170             :       // For UNDEF elements, we don't know anything about the common state of
    3171             :       // the shuffle result.
    3172        5173 :       if (M < 0)
    3173             :         return 1;
    3174        5111 :       if ((unsigned)M < NumElts)
    3175        4081 :         DemandedLHS.setBit((unsigned)M % NumElts);
    3176             :       else
    3177        1030 :         DemandedRHS.setBit((unsigned)M % NumElts);
    3178             :     }
    3179         929 :     Tmp = std::numeric_limits<unsigned>::max();
    3180         929 :     if (!!DemandedLHS)
    3181        1720 :       Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
    3182         929 :     if (!!DemandedRHS) {
    3183         710 :       Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
    3184         355 :       Tmp = std::min(Tmp, Tmp2);
    3185             :     }
    3186             :     // If we don't know anything, early out and try computeKnownBits fall-back.
    3187         929 :     if (Tmp == 1)
    3188             :       break;
    3189             :     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
    3190             :     return Tmp;
    3191             :   }
    3192             : 
    3193       24320 :   case ISD::BITCAST: {
    3194       24320 :     SDValue N0 = Op.getOperand(0);
    3195       24320 :     EVT SrcVT = N0.getValueType();
    3196             :     unsigned SrcBits = SrcVT.getScalarSizeInBits();
    3197             : 
    3198             :     // Ignore bitcasts from unsupported types..
    3199       24320 :     if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
    3200             :       break;
    3201             : 
    3202             :     // Fast handling of 'identity' bitcasts.
    3203       24311 :     if (VTBits == SrcBits)
    3204        6644 :       return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
    3205             : 
    3206             :     // Bitcast 'large element' scalar/vector to 'small element' vector.
    3207             :     // TODO: Handle cases other than 'sign splat' when we have a use case.
    3208             :     // Requires handling of DemandedElts and Endianness.
    3209       22455 :     if ((SrcBits % VTBits) == 0) {
    3210             :       assert(Op.getValueType().isVector() && "Expected bitcast to vector");
    3211       10191 :       Tmp = ComputeNumSignBits(N0, Depth + 1);
    3212       10191 :       if (Tmp == SrcBits)
    3213             :         return VTBits;
    3214             :     }
    3215             :     break;
    3216             :   }
    3217             : 
    3218        1783 :   case ISD::SIGN_EXTEND:
    3219        1783 :     Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
    3220        3566 :     return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
    3221        2949 :   case ISD::SIGN_EXTEND_INREG:
    3222             :     // Max of the input and what this extends.
    3223        5898 :     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
    3224        2949 :     Tmp = VTBits-Tmp+1;
    3225        5898 :     Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
    3226        2949 :     return std::max(Tmp, Tmp2);
    3227          57 :   case ISD::SIGN_EXTEND_VECTOR_INREG: {
    3228          57 :     SDValue Src = Op.getOperand(0);
    3229          57 :     EVT SrcVT = Src.getValueType();
    3230          57 :     APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());
    3231          57 :     Tmp = VTBits - SrcVT.getScalarSizeInBits();
    3232          57 :     return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
    3233             :   }
    3234             : 
    3235        1974 :   case ISD::SRA:
    3236        3948 :     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
    3237             :     // SRA X, C   -> adds C sign bits.
    3238        1974 :     if (ConstantSDNode *C =
    3239        3948 :             isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) {
    3240        1819 :       APInt ShiftVal = C->getAPIntValue();
    3241        1819 :       ShiftVal += Tmp;
    3242        5292 :       Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
    3243             :     }
    3244        1974 :     return Tmp;
    3245        5870 :   case ISD::SHL:
    3246        5870 :     if (ConstantSDNode *C =
    3247        5870 :             isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) {
    3248             :       // shl destroys sign bits.
    3249       10910 :       Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
    3250       21820 :       if (C->getAPIntValue().uge(VTBits) ||      // Bad shift.
    3251        5455 :           C->getAPIntValue().uge(Tmp)) break;    // Shifted all sign bits out.
    3252         307 :       return Tmp - C->getZExtValue();
    3253         415 :     }
    3254             :     break;
    3255       16922 :   case ISD::AND:
    3256             :   case ISD::OR:
    3257             :   case ISD::XOR:    // NOT is handled here.
    3258             :     // Logical binary ops preserve the number of sign bits at the worst.
    3259       33844 :     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
    3260       16922 :     if (Tmp != 1) {
    3261       12706 :       Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
    3262        6353 :       FirstAnswer = std::min(Tmp, Tmp2);
    3263             :       // We computed what we know about the sign bits as our first
    3264             :       // answer. Now proceed to the generic code that uses
    3265             :       // computeKnownBits, and pick whichever answer is better.
    3266             :     }
    3267             :     break;
    3268             : 
    3269         585 :   case ISD::SELECT:
    3270             :   case ISD::VSELECT:
    3271        1170 :     Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
    3272         585 :     if (Tmp == 1) return 1;  // Early out.
    3273         352 :     Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
    3274         176 :     return std::min(Tmp, Tmp2);
    3275         160 :   case ISD::SELECT_CC:
    3276         320 :     Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
    3277         160 :     if (Tmp == 1) return 1;  // Early out.
    3278         218 :     Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
    3279         109 :     return std::min(Tmp, Tmp2);
    3280             : 
    3281         701 :   case ISD::SMIN:
    3282             :   case ISD::SMAX: {
    3283             :     // If we have a clamp pattern, we know that the number of sign bits will be
    3284             :     // the minimum of the clamp min/max range.
    3285             :     bool IsMax = (Opcode == ISD::SMAX);
    3286             :     ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
    3287         701 :     if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)))
    3288        1755 :       if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
    3289         580 :         CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1),
    3290             :                                               DemandedElts);
    3291         701 :     if (CstLow && CstHigh) {
    3292         576 :       if (!IsMax)
    3293             :         std::swap(CstLow, CstHigh);
    3294        1728 :       if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
    3295         576 :         Tmp = CstLow->getAPIntValue().getNumSignBits();
    3296         576 :         Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
    3297         576 :         return std::min(Tmp, Tmp2);
    3298             :       }
    3299             :     }
    3300             : 
    3301             :     // Fallback - just get the minimum number of sign bits of the operands.
    3302         250 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
    3303         125 :     if (Tmp == 1)
    3304             :       return 1;  // Early out.
    3305         190 :     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
    3306          95 :     return std::min(Tmp, Tmp2);
    3307             :   }
    3308         531 :   case ISD::UMIN:
    3309             :   case ISD::UMAX:
    3310        1062 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
    3311         531 :     if (Tmp == 1)
    3312             :       return 1;  // Early out.
    3313          36 :     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
    3314          18 :     return std::min(Tmp, Tmp2);
    3315          93 :   case ISD::SADDO:
    3316             :   case ISD::UADDO:
    3317             :   case ISD::SSUBO:
    3318             :   case ISD::USUBO:
    3319             :   case ISD::SMULO:
    3320             :   case ISD::UMULO:
    3321          93 :     if (Op.getResNo() != 1)
    3322             :       break;
    3323             :     // The boolean result conforms to getBooleanContents.  Fall through.
    3324             :     // If setcc returns 0/-1, all bits are sign bits.
    3325             :     // We know that we have an integer-based boolean since these operations
    3326             :     // are only available for integer.
    3327         165 :     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
    3328             :         TargetLowering::ZeroOrNegativeOneBooleanContent)
    3329             :       return VTBits;
    3330             :     break;
    3331        5919 :   case ISD::SETCC:
    3332             :     // If setcc returns 0/-1, all bits are sign bits.
    3333       17757 :     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    3334             :         TargetLowering::ZeroOrNegativeOneBooleanContent)
    3335             :       return VTBits;
    3336             :     break;
    3337          56 :   case ISD::ROTL:
    3338             :   case ISD::ROTR:
    3339             :     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
    3340         112 :       unsigned RotAmt = C->getAPIntValue().urem(VTBits);
    3341             : 
    3342             :       // Handle rotate right by N like a rotate left by 32-N.
    3343          56 :       if (Opcode == ISD::ROTR)
    3344          28 :         RotAmt = (VTBits - RotAmt) % VTBits;
    3345             : 
    3346             :       // If we aren't rotating out all of the known-in sign bits, return the
    3347             :       // number that are left.  This handles rotl(sext(x), 1) for example.
    3348         112 :       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3349          56 :       if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
    3350             :     }
    3351             :     break;
    3352       19166 :   case ISD::ADD:
    3353             :   case ISD::ADDC:
    3354             :     // Add can have at most one carry bit.  Thus we know that the output
    3355             :     // is, at worst, one more bit than the inputs.
    3356       38332 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3357       19166 :     if (Tmp == 1) return 1;  // Early out.
    3358             : 
    3359             :     // Special case decrementing a value (ADD X, -1):
    3360        1152 :     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
    3361        1118 :       if (CRHS->isAllOnesValue()) {
    3362          11 :         KnownBits Known;
    3363          72 :         computeKnownBits(Op.getOperand(0), Known, Depth+1);
    3364             : 
    3365             :         // If the input is known to be 0 or 1, the output is 0/-1, which is all
    3366             :         // sign bits set.
    3367         216 :         if ((Known.Zero | 1).isAllOnesValue())
    3368          61 :           return VTBits;
    3369             : 
    3370             :         // If we are subtracting one from a positive number, there is no carry
    3371             :         // out of the result.
    3372          64 :         if (Known.isNonNegative())
    3373             :           return Tmp;
    3374             :       }
    3375             : 
    3376        2182 :     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
    3377        1091 :     if (Tmp2 == 1) return 1;
    3378        1013 :     return std::min(Tmp, Tmp2)-1;
    3379             : 
    3380        5331 :   case ISD::SUB:
    3381       10662 :     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
    3382        5331 :     if (Tmp2 == 1) return 1;
    3383             : 
    3384             :     // Handle NEG.
    3385        1508 :     if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0)))
    3386         346 :       if (CLHS->isNullValue()) {
    3387          56 :         KnownBits Known;
    3388         220 :         computeKnownBits(Op.getOperand(1), Known, Depth+1);
    3389             :         // If the input is known to be 0 or 1, the output is 0/-1, which is all
    3390             :         // sign bits set.
    3391         330 :         if ((Known.Zero | 1).isAllOnesValue())
    3392          54 :           return VTBits;
    3393             : 
    3394             :         // If the input is known to be positive (the sign bit is known clear),
    3395             :         // the output of the NEG has the same number of sign bits as the input.
    3396         105 :         if (Known.isNonNegative())
    3397             :           return Tmp2;
    3398             : 
    3399             :         // Otherwise, we treat this like a SUB.
    3400             :       }
    3401             : 
    3402             :     // Sub can have at most one carry bit.  Thus we know that the output
    3403             :     // is, at worst, one more bit than the inputs.
    3404        1400 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3405         700 :     if (Tmp == 1) return 1;  // Early out.
    3406         464 :     return std::min(Tmp, Tmp2)-1;
    3407       33891 :   case ISD::TRUNCATE: {
    3408             :     // Check if the sign bits of source go down as far as the truncated value.
    3409       33891 :     unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
    3410       67782 :     unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
    3411       33891 :     if (NumSrcSignBits > (NumSrcBits - VTBits))
    3412        5360 :       return NumSrcSignBits - (NumSrcBits - VTBits);
    3413             :     break;
    3414             :   }
    3415          30 :   case ISD::EXTRACT_ELEMENT: {
    3416          60 :     const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3417          30 :     const int BitWidth = Op.getValueSizeInBits();
    3418          60 :     const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
    3419             : 
    3420             :     // Get reverse index (starting from 1), Op1 value indexes elements from
    3421             :     // little end. Sign starts at big end.
    3422          60 :     const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
    3423             : 
    3424             :     // If the sign portion ends in our element the subtraction gives correct
    3425             :     // result. Otherwise it gives either negative or > bitwidth result
    3426          60 :     return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
    3427             :   }
    3428         140 :   case ISD::INSERT_VECTOR_ELT: {
    3429         140 :     SDValue InVec = Op.getOperand(0);
    3430         140 :     SDValue InVal = Op.getOperand(1);
    3431         140 :     SDValue EltNo = Op.getOperand(2);
    3432         140 :     unsigned NumElts = InVec.getValueType().getVectorNumElements();
    3433             : 
    3434             :     ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
    3435         264 :     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
    3436             :       // If we know the element index, split the demand between the
    3437             :       // source vector and the inserted element.
    3438         132 :       unsigned EltIdx = CEltNo->getZExtValue();
    3439             : 
    3440             :       // If we demand the inserted element then get its sign bits.
    3441         132 :       Tmp = std::numeric_limits<unsigned>::max();
    3442         132 :       if (DemandedElts[EltIdx]) {
    3443             :         // TODO - handle implicit truncation of inserted elements.
    3444         117 :         if (InVal.getScalarValueSizeInBits() != VTBits)
    3445             :           break;
    3446         117 :         Tmp = ComputeNumSignBits(InVal, Depth + 1);
    3447             :       }
    3448             : 
    3449             :       // If we demand the source vector then get its sign bits, and determine
    3450             :       // the minimum.
    3451             :       APInt VectorElts = DemandedElts;
    3452             :       VectorElts.clearBit(EltIdx);
    3453         132 :       if (!!VectorElts) {
    3454          94 :         Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1);
    3455          94 :         Tmp = std::min(Tmp, Tmp2);
    3456             :       }
    3457             :     } else {
    3458             :       // Unknown element index, so ignore DemandedElts and demand them all.
    3459           8 :       Tmp = ComputeNumSignBits(InVec, Depth + 1);
    3460           8 :       Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
    3461           8 :       Tmp = std::min(Tmp, Tmp2);
    3462             :     }
    3463             :     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
    3464         140 :     return Tmp;
    3465             :   }
    3466       17355 :   case ISD::EXTRACT_VECTOR_ELT: {
    3467       17355 :     SDValue InVec = Op.getOperand(0);
    3468       17355 :     SDValue EltNo = Op.getOperand(1);
    3469       17355 :     EVT VecVT = InVec.getValueType();
    3470       17355 :     const unsigned BitWidth = Op.getValueSizeInBits();
    3471       34710 :     const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
    3472       17355 :     const unsigned NumSrcElts = VecVT.getVectorNumElements();
    3473             : 
    3474             :     // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
    3475             :     // anything about sign bits. But if the sizes match we can derive knowledge
    3476             :     // about sign bits from the vector operand.
    3477       17355 :     if (BitWidth != EltBitWidth)
    3478             :       break;
    3479             : 
    3480             :     // If we know the element index, just demand that vector element, else for
    3481             :     // an unknown element index, ignore DemandedElts and demand them all.
    3482       16376 :     APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
    3483             :     ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
    3484       32752 :     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
    3485             :       DemandedSrcElts =
    3486       32752 :           APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
    3487             : 
    3488       16376 :     return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
    3489             :   }
    3490        9701 :   case ISD::EXTRACT_SUBVECTOR: {
    3491             :     // If we know the element index, just demand that subvector elements,
    3492             :     // otherwise demand them all.
    3493        9701 :     SDValue Src = Op.getOperand(0);
    3494             :     ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
    3495        9701 :     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
    3496       29103 :     if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
    3497             :       // Offset the demanded elts by the subvector index.
    3498             :       uint64_t Idx = SubIdx->getZExtValue();
    3499       19402 :       APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx);
    3500        9701 :       return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
    3501             :     }
    3502           0 :     return ComputeNumSignBits(Src, Depth + 1);
    3503             :   }
    3504        2632 :   case ISD::CONCAT_VECTORS:
    3505             :     // Determine the minimum number of sign bits across all demanded
    3506             :     // elts of the input vectors. Early out if the result is already 1.
    3507        2632 :     Tmp = std::numeric_limits<unsigned>::max();
    3508        5264 :     EVT SubVectorVT = Op.getOperand(0).getValueType();
    3509        2632 :     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
    3510             :     unsigned NumSubVectors = Op.getNumOperands();
    3511       10534 :     for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
    3512        3951 :       APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
    3513        7902 :       DemandedSub = DemandedSub.trunc(NumSubVectorElts);
    3514        3951 :       if (!DemandedSub)
    3515             :         continue;
    3516        5706 :       Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
    3517        2853 :       Tmp = std::min(Tmp, Tmp2);
    3518        2632 :     }
    3519             :     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
    3520        2632 :     return Tmp;
    3521             :   }
    3522             : 
    3523             :   // If we are looking at the loaded value of the SDNode.
    3524      271849 :   if (Op.getResNo() == 0) {
    3525             :     // Handle LOADX separately here. EXTLOAD case will fallthrough.
    3526             :     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
    3527             :       unsigned ExtType = LD->getExtensionType();
    3528       47038 :       switch (ExtType) {
    3529             :         default: break;
    3530        1348 :         case ISD::SEXTLOAD:    // '17' bits known
    3531        2696 :           Tmp = LD->getMemoryVT().getScalarSizeInBits();
    3532        1348 :           return VTBits-Tmp+1;
    3533        1635 :         case ISD::ZEXTLOAD:    // '16' bits known
    3534        3270 :           Tmp = LD->getMemoryVT().getScalarSizeInBits();
    3535        1635 :           return VTBits-Tmp;
    3536             :       }
    3537             :     }
    3538             :   }
    3539             : 
    3540             :   // Allow the target to implement this method for its nodes.
    3541      537732 :   if (Opcode >= ISD::BUILTIN_OP_END ||
    3542      268866 :       Opcode == ISD::INTRINSIC_WO_CHAIN ||
    3543      248810 :       Opcode == ISD::INTRINSIC_W_CHAIN ||
    3544             :       Opcode == ISD::INTRINSIC_VOID) {
    3545             :     unsigned NumBits =
    3546       20723 :         TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
    3547       20723 :     if (NumBits > 1)
    3548       11010 :       FirstAnswer = std::max(FirstAnswer, NumBits);
    3549             :   }
    3550             : 
    3551             :   // Finally, if we can prove that the top bits of the result are 0's or 1's,
    3552             :   // use this information.
    3553      268866 :   KnownBits Known;
    3554      268866 :   computeKnownBits(Op, Known, DemandedElts, Depth);
    3555             : 
    3556             :   APInt Mask;
    3557      268866 :   if (Known.isNonNegative()) {        // sign bit is 0
    3558       47118 :     Mask = Known.Zero;
    3559      221748 :   } else if (Known.isNegative()) {  // sign bit is 1;
    3560          62 :     Mask = Known.One;
    3561             :   } else {
    3562             :     // Nothing known.
    3563      221686 :     return FirstAnswer;
    3564             :   }
    3565             : 
    3566             :   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
    3567             :   // the number of identical bits in the top of the input value.
    3568       47180 :   Mask = ~Mask;
    3569       47180 :   Mask <<= Mask.getBitWidth()-VTBits;
    3570             :   // Return # leading zeros.  We use 'min' here in case Val was zero before
    3571             :   // shifting.  We don't want to return '64' as for an i32 "0".
    3572       94360 :   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
    3573             : }
    3574             : 
    3575     2917519 : bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
    3576     2917519 :   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
    3577             :       !isa<ConstantSDNode>(Op.getOperand(1)))
    3578             :     return false;
    3579             : 
    3580     2034848 :   if (Op.getOpcode() == ISD::OR &&
    3581      398312 :       !MaskedValueIsZero(Op.getOperand(0),
    3582             :                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
    3583             :     return false;
    3584             : 
    3585             :   return true;
    3586             : }
    3587             : 
    3588         599 : bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
    3589             :   // If we're told that NaNs won't happen, assume they won't.
    3590         599 :   if (getTarget().Options.NoNaNsFPMath)
    3591             :     return true;
    3592             : 
    3593         405 :   if (Op->getFlags().hasNoNaNs())
    3594             :     return true;
    3595             : 
    3596             :   // If the value is a constant, we can obviously see if it is a NaN or not.
    3597             :   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
    3598          90 :     return !C->getValueAPF().isNaN();
    3599             : 
    3600             :   // TODO: Recognize more cases here.
    3601             : 
    3602             :   return false;
    3603             : }
    3604             : 
    3605         851 : bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
    3606             :   // If the value is a constant, we can obviously see if it is a zero or not.
    3607             :   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
    3608          52 :     return !C->isZero();
    3609             : 
    3610             :   // TODO: Recognize more cases here.
    3611         825 :   switch (Op.getOpcode()) {
    3612             :   default: break;
    3613          38 :   case ISD::OR:
    3614             :     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
    3615          42 :       return !C->isNullValue();
    3616             :     break;
    3617             :   }
    3618             : 
    3619             :   return false;
    3620             : }
    3621             : 
    3622       11180 : bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
    3623             :   // Check the obvious case.
    3624             :   if (A == B) return true;
    3625             : 
    3626             :   // For for negative and positive zero.
    3627             :   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
    3628             :     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
    3629          34 :       if (CA->isZero() && CB->isZero()) return true;
    3630             : 
    3631             :   // Otherwise they may not be equal.
    3632             :   return false;
    3633             : }
    3634             : 
    3635     1783499 : bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
    3636             :   assert(A.getValueType() == B.getValueType() &&
    3637             :          "Values must have the same type");
    3638     1783499 :   KnownBits AKnown, BKnown;
    3639     1783499 :   computeKnownBits(A, AKnown);
    3640     1783499 :   computeKnownBits(B, BKnown);
    3641     5350497 :   return (AKnown.Zero | BKnown.Zero).isAllOnesValue();
    3642             : }
    3643             : 
    3644       39269 : static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
    3645             :                                   ArrayRef<SDValue> Ops,
    3646             :                                   SelectionDAG &DAG) {
    3647             :   assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
    3648             :   assert(llvm::all_of(Ops,
    3649             :                       [Ops](SDValue Op) {
    3650             :                         return Ops[0].getValueType() == Op.getValueType();
    3651             :                       }) &&
    3652             :          "Concatenation of vectors with inconsistent value types!");
    3653             :   assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==
    3654             :              VT.getVectorNumElements() &&
    3655             :          "Incorrect element count in vector concatenation!");
    3656             : 
    3657       39269 :   if (Ops.size() == 1)
    3658           0 :     return Ops[0];
    3659             : 
    3660             :   // Concat of UNDEFs is UNDEF.
    3661       39269 :   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
    3662         211 :     return DAG.getUNDEF(VT);
    3663             : 
    3664             :   // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
    3665             :   // simplified to one big BUILD_VECTOR.
    3666             :   // FIXME: Add support for SCALAR_TO_VECTOR as well.
    3667       39058 :   EVT SVT = VT.getScalarType();
    3668             :   SmallVector<SDValue, 16> Elts;
    3669       46548 :   for (SDValue Op : Ops) {
    3670       41836 :     EVT OpVT = Op.getValueType();
    3671       41836 :     if (Op.isUndef())
    3672         479 :       Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
    3673       41357 :     else if (Op.getOpcode() == ISD::BUILD_VECTOR)
    3674        6532 :       Elts.append(Op->op_begin(), Op->op_end());
    3675             :     else
    3676       38091 :       return SDValue();
    3677             :   }
    3678             : 
    3679             :   // BUILD_VECTOR requires all inputs to be of the same type, find the
    3680             :   // maximum type and extend them all.
    3681       33691 :   for (SDValue Op : Elts)
    3682       16362 :     SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
    3683             : 
    3684         967 :   if (SVT.bitsGT(VT.getScalarType()))
    3685       25911 :     for (SDValue &Op : Elts)
    3686       38004 :       Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
    3687       25336 :                ? DAG.getZExtOrTrunc(Op, DL, SVT)
    3688             :                : DAG.getSExtOrTrunc(Op, DL, SVT);
    3689             : 
    3690         967 :   SDValue V = DAG.getBuildVector(VT, DL, Elts);
    3691             :   NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
    3692         967 :   return V;
    3693             : }
    3694             : 
    3695             : /// Gets or creates the specified node.
    3696     2528448 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
    3697             :   FoldingSetNodeID ID;
    3698     2528448 :   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
    3699     2528448 :   void *IP = nullptr;
    3700     2528448 :   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    3701     2141727 :     return SDValue(E, 0);
    3702             : 
    3703      386721 :   auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
    3704      773442 :                               getVTList(VT));
    3705      386721 :   CSEMap.InsertNode(N, IP);
    3706             : 
    3707      386721 :   InsertNode(N);
    3708             :   SDValue V = SDValue(N, 0);
    3709             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    3710      386721 :   return V;
    3711             : }
    3712             : 
    3713     3738759 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    3714             :                               SDValue Operand, const SDNodeFlags Flags) {
    3715             :   // Constant fold unary operations with an integer constant operand. Even
    3716             :   // opaque constant will be folded, because the folding of unary operations
    3717             :   // doesn't create new constants with different values. Nevertheless, the
    3718             :   // opaque flag is preserved during folding to prevent future folding with
    3719             :   // other constants.
    3720             :   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
    3721      279613 :     const APInt &Val = C->getAPIntValue();
    3722      279613 :     switch (Opcode) {
    3723             :     default: break;
    3724       34166 :     case ISD::SIGN_EXTEND:
    3725       68332 :       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
    3726      102498 :                          C->isTargetOpcode(), C->isOpaque());
    3727      173232 :     case ISD::ANY_EXTEND:
    3728             :     case ISD::ZERO_EXTEND:
    3729             :     case ISD::TRUNCATE:
    3730      346464 :       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
    3731      519696 :                          C->isTargetOpcode(), C->isOpaque());
    3732         365 :     case ISD::UINT_TO_FP:
    3733             :     case ISD::SINT_TO_FP: {
    3734             :       APFloat apf(EVTToAPFloatSemantics(VT),
    3735         730 :                   APInt::getNullValue(VT.getSizeInBits()));
    3736         365 :       (void)apf.convertFromAPInt(Val,
    3737             :                                  Opcode==ISD::SINT_TO_FP,
    3738             :                                  APFloat::rmNearestTiesToEven);
    3739         365 :       return getConstantFP(apf, DL, VT);
    3740             :     }
    3741             :     case ISD::BITCAST:
    3742           9 :       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
    3743          27 :         return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
    3744         351 :       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
    3745        1053 :         return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
    3746         113 :       if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
    3747         339 :         return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
    3748           0 :       if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
    3749           0 :         return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
    3750             :       break;
    3751          14 :     case ISD::ABS:
    3752          28 :       return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
    3753          42 :                          C->isOpaque());
    3754         452 :     case ISD::BITREVERSE:
    3755         904 :       return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
    3756        1356 :                          C->isOpaque());
    3757         170 :     case ISD::BSWAP:
    3758         340 :       return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
    3759         510 :                          C->isOpaque());
    3760         660 :     case ISD::CTPOP:
    3761             :       return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
    3762        1980 :                          C->isOpaque());
    3763       29025 :     case ISD::CTLZ:
    3764             :     case ISD::CTLZ_ZERO_UNDEF:
    3765       29025 :       return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
    3766       87075 :                          C->isOpaque());
    3767        1860 :     case ISD::CTTZ:
    3768             :     case ISD::CTTZ_ZERO_UNDEF:
    3769        1860 :       return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
    3770        5580 :                          C->isOpaque());
    3771         353 :     case ISD::FP16_TO_FP: {
    3772             :       bool Ignored;
    3773             :       APFloat FPV(APFloat::IEEEhalf(),
    3774         706 :                   (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
    3775             : 
    3776             :       // This can return overflow, underflow, or inexact; we don't care.
    3777             :       // FIXME need to be more flexible about rounding mode.
    3778         353 :       (void)FPV.convert(EVTToAPFloatSemantics(VT),
    3779             :                         APFloat::rmNearestTiesToEven, &Ignored);
    3780         353 :       return getConstantFP(FPV, DL, VT);
    3781             :     }
    3782             :     }
    3783             :   }
    3784             : 
    3785             :   // Constant fold unary operations with a floating point constant operand.
    3786             :   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
    3787        5035 :     APFloat V = C->getValueAPF();    // make copy
    3788        5035 :     switch (Opcode) {
    3789         210 :     case ISD::FNEG:
    3790         210 :       V.changeSign();
    3791         210 :       return getConstantFP(V, DL, VT);
    3792          12 :     case ISD::FABS:
    3793          12 :       V.clearSign();
    3794          12 :       return getConstantFP(V, DL, VT);
    3795          22 :     case ISD::FCEIL: {
    3796          22 :       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
    3797          22 :       if (fs == APFloat::opOK || fs == APFloat::opInexact)
    3798          22 :         return getConstantFP(V, DL, VT);
    3799             :       break;
    3800             :     }
    3801          19 :     case ISD::FTRUNC: {
    3802          19 :       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
    3803          19 :       if (fs == APFloat::opOK || fs == APFloat::opInexact)
    3804          19 :         return getConstantFP(V, DL, VT);
    3805             :       break;
    3806             :     }
    3807          19 :     case ISD::FFLOOR: {
    3808          19 :       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
    3809          19 :       if (fs == APFloat::opOK || fs == APFloat::opInexact)
    3810          19 :         return getConstantFP(V, DL, VT);
    3811             :       break;
    3812             :     }
    3813         104 :     case ISD::FP_EXTEND: {
    3814             :       bool ignored;
    3815             :       // This can return overflow, underflow, or inexact; we don't care.
    3816             :       // FIXME need to be more flexible about rounding mode.
    3817         104 :       (void)V.convert(EVTToAPFloatSemantics(VT),
    3818             :                       APFloat::rmNearestTiesToEven, &ignored);
    3819         104 :       return getConstantFP(V, DL, VT);
    3820             :     }
    3821         625 :     case ISD::FP_TO_SINT:
    3822             :     case ISD::FP_TO_UINT: {
    3823             :       bool ignored;
    3824         625 :       APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
    3825             :       // FIXME need to be more flexible about rounding mode.
    3826             :       APFloat::opStatus s =
    3827         625 :           V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
    3828         625 :       if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
    3829             :         break;
    3830         597 :       return getConstant(IntVal, DL, VT);
    3831             :     }
    3832             :     case ISD::BITCAST:
    3833         193 :       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
    3834         579 :         return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
    3835        2463 :       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
    3836        7389 :         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
    3837         287 :       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
    3838         861 :         return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
    3839             :       break;
    3840          48 :     case ISD::FP_TO_FP16: {
    3841             :       bool Ignored;
    3842             :       // This can return overflow, underflow, or inexact; we don't care.
    3843             :       // FIXME need to be more flexible about rounding mode.
    3844          48 :       (void)V.convert(APFloat::IEEEhalf(),
    3845             :                       APFloat::rmNearestTiesToEven, &Ignored);
    3846          96 :       return getConstant(V.bitcastToAPInt(), DL, VT);
    3847             :     }
    3848             :     }
    3849             :   }
    3850             : 
    3851             :   // Constant fold unary operations with a vector integer or float operand.
    3852             :   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
    3853      166481 :     if (BV->isConstant()) {
    3854             :       switch (Opcode) {
    3855             :       default:
    3856             :         // FIXME: Entirely reasonable to perform folding of other unary
    3857             :         // operations here as the need arises.
    3858             :         break;
    3859        1814 :       case ISD::FNEG:
    3860             :       case ISD::FABS:
    3861             :       case ISD::FCEIL:
    3862             :       case ISD::FTRUNC:
    3863             :       case ISD::FFLOOR:
    3864             :       case ISD::FP_EXTEND:
    3865             :       case ISD::FP_TO_SINT:
    3866             :       case ISD::FP_TO_UINT:
    3867             :       case ISD::TRUNCATE:
    3868             :       case ISD::ANY_EXTEND:
    3869             :       case ISD::ZERO_EXTEND:
    3870             :       case ISD::SIGN_EXTEND:
    3871             :       case ISD::UINT_TO_FP:
    3872             :       case ISD::SINT_TO_FP:
    3873             :       case ISD::ABS:
    3874             :       case ISD::BITREVERSE:
    3875             :       case ISD::BSWAP:
    3876             :       case ISD::CTLZ:
    3877             :       case ISD::CTLZ_ZERO_UNDEF:
    3878             :       case ISD::CTTZ:
    3879             :       case ISD::CTTZ_ZERO_UNDEF:
    3880             :       case ISD::CTPOP: {
    3881        1814 :         SDValue Ops = { Operand };
    3882        1814 :         if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
    3883        1794 :           return Fold;
    3884             :       }
    3885             :       }
    3886             :     }
    3887             :   }
    3888             : 
    3889     3492221 :   unsigned OpOpcode = Operand.getNode()->getOpcode();
    3890     3492221 :   switch (Opcode) {
    3891     1208538 :   case ISD::TokenFactor:
    3892             :   case ISD::MERGE_VALUES:
    3893             :   case ISD::CONCAT_VECTORS:
    3894     1208538 :     return Operand;         // Factor, merge or concat of one node?  No need.
    3895           0 :   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
    3896        9649 :   case ISD::FP_EXTEND:
    3897             :     assert(VT.isFloatingPoint() &&
    3898             :            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
    3899        9690 :     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
    3900             :     assert((!VT.isVector() ||
    3901             :             VT.getVectorNumElements() ==
    3902             :             Operand.getValueType().getVectorNumElements()) &&
    3903             :            "Vector element count mismatch!");
    3904             :     assert(Operand.getValueType().bitsLT(VT) &&
    3905             :            "Invalid fpext node, dst < src!");
    3906        7194 :     if (Operand.isUndef())
    3907          41 :       return getUNDEF(VT);
    3908             :     break;
    3909       54352 :   case ISD::SIGN_EXTEND:
    3910             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3911             :            "Invalid SIGN_EXTEND!");
    3912       54431 :     if (Operand.getValueType() == VT) return Operand;   // noop extension
    3913             :     assert((!VT.isVector() ||
    3914             :             VT.getVectorNumElements() ==
    3915             :             Operand.getValueType().getVectorNumElements()) &&
    3916             :            "Vector element count mismatch!");
    3917             :     assert(Operand.getValueType().bitsLT(VT) &&
    3918             :            "Invalid sext node, dst < src!");
    3919       51741 :     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
    3920         444 :       return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
    3921       51297 :     else if (OpOpcode == ISD::UNDEF)
    3922             :       // sext(undef) = 0, because the top bits will all be the same.
    3923         217 :       return getConstant(0, DL, VT);
    3924             :     break;
    3925      165935 :   case ISD::ZERO_EXTEND:
    3926             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3927             :            "Invalid ZERO_EXTEND!");
    3928      167715 :     if (Operand.getValueType() == VT) return Operand;   // noop extension
    3929             :     assert((!VT.isVector() ||
    3930             :             VT.getVectorNumElements() ==
    3931             :             Operand.getValueType().getVectorNumElements()) &&
    3932             :            "Vector element count mismatch!");
    3933             :     assert(Operand.getValueType().bitsLT(VT) &&
    3934             :            "Invalid zext node, dst < src!");
    3935      161284 :     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
    3936         446 :       return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
    3937      160838 :     else if (OpOpcode == ISD::UNDEF)
    3938             :       // zext(undef) = 0, because the top bits will be zero.
    3939         815 :       return getConstant(0, DL, VT);
    3940             :     break;
    3941      151274 :   case ISD::ANY_EXTEND:
    3942             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3943             :            "Invalid ANY_EXTEND!");
    3944      152935 :     if (Operand.getValueType() == VT) return Operand;   // noop extension
    3945             :     assert((!VT.isVector() ||
    3946             :             VT.getVectorNumElements() ==
    3947             :             Operand.getValueType().getVectorNumElements()) &&
    3948             :            "Vector element count mismatch!");
    3949             :     assert(Operand.getValueType().bitsLT(VT) &&
    3950             :            "Invalid anyext node, dst < src!");
    3951             : 
    3952       88696 :     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
    3953             :         OpOpcode == ISD::ANY_EXTEND)
    3954             :       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
    3955        2485 :       return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
    3956       86211 :     else if (OpOpcode == ISD::UNDEF)
    3957         353 :       return getUNDEF(VT);
    3958             : 
    3959             :     // (ext (trunx x)) -> x
    3960       85858 :     if (OpOpcode == ISD::TRUNCATE) {
    3961       13077 :       SDValue OpOp = Operand.getOperand(0);
    3962           6 :       if (OpOp.getValueType() == VT)
    3963        9896 :         return OpOp;
    3964             :     }
    3965             :     break;
    3966      615116 :   case ISD::TRUNCATE:
    3967             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3968             :            "Invalid TRUNCATE!");
    3969      620039 :     if (Operand.getValueType() == VT) return Operand;   // noop truncate
    3970             :     assert((!VT.isVector() ||
    3971             :             VT.getVectorNumElements() ==
    3972             :             Operand.getValueType().getVectorNumElements()) &&
    3973             :            "Vector element count mismatch!");
    3974             :     assert(Operand.getValueType().bitsGT(VT) &&
    3975             :            "Invalid truncate node, src < dst!");
    3976      340237 :     if (OpOpcode == ISD::TRUNCATE)
    3977       35928 :       return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
    3978      304309 :     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
    3979             :         OpOpcode == ISD::ANY_EXTEND) {
    3980             :       // If the source is smaller than the dest, we still need an extend.
    3981       34956 :       if (Operand.getOperand(0).getValueType().getScalarType()
    3982       11652 :             .bitsLT(VT.getScalarType()))
    3983        4604 :         return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
    3984       14096 :       if (Operand.getOperand(0).getValueType().bitsGT(VT))
    3985         259 :         return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
    3986        6789 :       return Operand.getOperand(0);
    3987             :     }
    3988      292657 :     if (OpOpcode == ISD::UNDEF)
    3989       12774 :       return getUNDEF(VT);
    3990             :     break;
    3991         414 :   case ISD::ABS:
    3992             :     assert(VT.isInteger() && VT == Operand.getValueType() &&
    3993             :            "Invalid ABS!");
    3994         414 :     if (OpOpcode == ISD::UNDEF)
    3995           1 :       return getUNDEF(VT);
    3996             :     break;
    3997         956 :   case ISD::BSWAP:
    3998             :     assert(VT.isInteger() && VT == Operand.getValueType() &&
    3999             :            "Invalid BSWAP!");
    4000             :     assert((VT.getScalarSizeInBits() % 16 == 0) &&
    4001             :            "BSWAP types must be a multiple of 16 bits!");
    4002         956 :     if (OpOpcode == ISD::UNDEF)
    4003           4 :       return getUNDEF(VT);
    4004             :     break;
    4005         698 :   case ISD::BITREVERSE:
    4006             :     assert(VT.isInteger() && VT == Operand.getValueType() &&
    4007             :            "Invalid BITREVERSE!");
    4008         698 :     if (OpOpcode == ISD::UNDEF)
    4009           4 :       return getUNDEF(VT);
    4010             :     break;
    4011      704375 :   case ISD::BITCAST:
    4012             :     // Basic sanity checking.
    4013             :     assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
    4014             :            "Cannot BITCAST between types of different sizes!");
    4015        4565 :     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
    4016      537688 :     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
    4017       39669 :       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
    4018      498019 :     if (OpOpcode == ISD::UNDEF)
    4019        8674 :       return getUNDEF(VT);
    4020             :     break;
    4021       17873 :   case ISD::SCALAR_TO_VECTOR:
    4022             :     assert(VT.isVector() && !Operand.getValueType().isVector() &&
    4023             :            (VT.getVectorElementType() == Operand.getValueType() ||
    4024             :             (VT.getVectorElementType().isInteger() &&
    4025             :              Operand.getValueType().isInteger() &&
    4026             :              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
    4027             :            "Illegal SCALAR_TO_VECTOR node!");
    4028       17873 :     if (OpOpcode == ISD::UNDEF)
    4029           1 :       return getUNDEF(VT);
    4030             :     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
    4031             :     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
    4032        1354 :         isa<ConstantSDNode>(Operand.getOperand(1)) &&
    4033       17872 :         Operand.getConstantOperandVal(1) == 0 &&
    4034        1786 :         Operand.getOperand(0).getValueType() == VT)
    4035         713 :       return Operand.getOperand(0);
    4036             :     break;
    4037        4312 :   case ISD::FNEG:
    4038             :     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
    4039        4312 :     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
    4040             :       // FIXME: FNEG has no fast-math-flags to propagate; use the FSUB's flags?
    4041             :       return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1),
    4042          59 :                      Operand.getOperand(0), Operand.getNode()->getFlags());
    4043        4253 :     if (OpOpcode == ISD::FNEG)  // --X -> X
    4044           0 :       return Operand.getOperand(0);
    4045             :     break;
    4046        1399 :   case ISD::FABS:
    4047        1399 :     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
    4048           4 :       return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
    4049             :     break;
    4050             :   }
    4051             : 
    4052             :   SDNode *N;
    4053     1645642 :   SDVTList VTs = getVTList(VT);
    4054     1645642 :   SDValue Ops[] = {Operand};
    4055     1645642 :   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
    4056             :     FoldingSetNodeID ID;
    4057     1645251 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    4058     1645251 :     void *IP = nullptr;
    4059     1645251 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
    4060       80486 :       E->intersectFlagsWith(Flags);
    4061       80486 :       return SDValue(E, 0);
    4062             :     }
    4063             : 
    4064     3129530 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4065             :     N->setFlags(Flags);
    4066     1564765 :     createOperands(N, Ops);
    4067     1564765 :     CSEMap.InsertNode(N, IP);
    4068             :   } else {
    4069         782 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4070         391 :     createOperands(N, Ops);
    4071             :   }
    4072             : 
    4073     1565156 :   InsertNode(N);
    4074             :   SDValue V = SDValue(N, 0);
    4075             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    4076     1565156 :   return V;
    4077             : }
    4078             : 
    4079      491410 : static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
    4080             :                                         const APInt &C2) {
    4081      491410 :   switch (Opcode) {
    4082      507982 :   case ISD::ADD:  return std::make_pair(C1 + C2, true);
    4083       57502 :   case ISD::SUB:  return std::make_pair(C1 - C2, true);
    4084        7668 :   case ISD::MUL:  return std::make_pair(C1 * C2, true);
    4085       23984 :   case ISD::AND:  return std::make_pair(C1 & C2, true);
    4086        2682 :   case ISD::OR:   return std::make_pair(C1 | C2, true);
    4087        4088 :   case ISD::XOR:  return std::make_pair(C1 ^ C2, true);
    4088      128244 :   case ISD::SHL:  return std::make_pair(C1 << C2, true);
    4089       28210 :   case ISD::SRL:  return std::make_pair(C1.lshr(C2), true);
    4090        1612 :   case ISD::SRA:  return std::make_pair(C1.ashr(C2), true);
    4091          32 :   case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
    4092           0 :   case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
    4093        1624 :   case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true);
    4094        1632 :   case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
    4095        1624 :   case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
    4096        1626 :   case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
    4097             :   case ISD::UDIV:
    4098          28 :     if (!C2.getBoolValue())
    4099             :       break;
    4100          56 :     return std::make_pair(C1.udiv(C2), true);
    4101             :   case ISD::UREM:
    4102          34 :     if (!C2.getBoolValue())
    4103             :       break;
    4104          68 :     return std::make_pair(C1.urem(C2), true);
    4105             :   case ISD::SDIV:
    4106          99 :     if (!C2.getBoolValue())
    4107             :       break;
    4108         198 :     return std::make_pair(C1.sdiv(C2), true);
    4109             :   case ISD::SREM:
    4110          57 :     if (!C2.getBoolValue())
    4111             :       break;
    4112         114 :     return std::make_pair(C1.srem(C2), true);
    4113             :   }
    4114      106937 :   return std::make_pair(APInt(1, 0), false);
    4115             : }
    4116             : 
    4117      491517 : SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
    4118             :                                              EVT VT, const ConstantSDNode *Cst1,
    4119             :                                              const ConstantSDNode *Cst2) {
    4120      982928 :   if (Cst1->isOpaque() || Cst2->isOpaque())
    4121         107 :     return SDValue();
    4122             : 
    4123             :   std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
    4124     1474230 :                                             Cst2->getAPIntValue());
    4125      491410 :   if (!Folded.second)
    4126      106937 :     return SDValue();
    4127      384473 :   return getConstant(Folded.first, DL, VT);
    4128             : }
    4129             : 
    4130      261622 : SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
    4131             :                                        const GlobalAddressSDNode *GA,
    4132             :                                        const SDNode *N2) {
    4133      261622 :   if (GA->getOpcode() != ISD::GlobalAddress)
    4134          12 :     return SDValue();
    4135      261610 :   if (!TLI->isOffsetFoldingLegal(GA))
    4136      255962 :     return SDValue();
    4137             :   const ConstantSDNode *Cst2 = dyn_cast<ConstantSDNode>(N2);
    4138             :   if (!Cst2)
    4139        1724 :     return SDValue();
    4140        3924 :   int64_t Offset = Cst2->getSExtValue();
    4141        3924 :   switch (Opcode) {
    4142             :   case ISD::ADD: break;
    4143           1 :   case ISD::SUB: Offset = -uint64_t(Offset); break;
    4144           8 :   default: return SDValue();
    4145             :   }
    4146        3916 :   return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT,
    4147        7832 :                           GA->getOffset() + uint64_t(Offset));
    4148             : }
    4149             : 
    4150     5087910 : bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
    4151     5087910 :   switch (Opcode) {
    4152       15742 :   case ISD::SDIV:
    4153             :   case ISD::UDIV:
    4154             :   case ISD::SREM:
    4155             :   case ISD::UREM: {
    4156             :     // If a divisor is zero/undef or any element of a divisor vector is
    4157             :     // zero/undef, the whole op is undef.
    4158             :     assert(Ops.size() == 2 && "Div/rem should have 2 operands");
    4159       15742 :     SDValue Divisor = Ops[1];
    4160       15742 :     if (Divisor.isUndef() || isNullConstant(Divisor))
    4161             :       return true;
    4162             : 
    4163       17269 :     return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
    4164             :            llvm::any_of(Divisor->op_values(),
    4165       32518 :                         [](SDValue V) { return V.isUndef() ||
    4166       16250 :                                         isNullConstant(V); });
    4167             :     // TODO: Handle signed overflow.
    4168             :   }
    4169             :   // TODO: Handle oversized shifts.
    4170             :   default:
    4171             :     return false;
    4172             :   }
    4173             : }
    4174             : 
    4175     4627456 : SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
    4176             :                                              EVT VT, SDNode *Cst1,
    4177             :                                              SDNode *Cst2) {
    4178             :   // If the opcode is a target-specific ISD node, there's nothing we can
    4179             :   // do here and the operand rules may not line up with the below, so
    4180             :   // bail early.
    4181     4627456 :   if (Opcode >= ISD::BUILTIN_OP_END)
    4182      343198 :     return SDValue();
    4183             : 
    4184     4284258 :   if (isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)}))
    4185          90 :     return getUNDEF(VT);
    4186             : 
    4187             :   // Handle the case of two scalars.
    4188             :   if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
    4189             :     if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
    4190      484215 :       SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2);
    4191             :       assert((!Folded || !VT.isVector()) &&
    4192             :              "Can't fold vectors ops with scalar operands");
    4193      484215 :       return Folded;
    4194             :     }
    4195             :   }
    4196             : 
    4197             :   // fold (add Sym, c) -> Sym+c
    4198             :   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst1))
    4199      260323 :     return FoldSymbolOffset(Opcode, VT, GA, Cst2);
    4200     3539630 :   if (TLI->isCommutativeBinOp(Opcode))
    4201             :     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst2))
    4202        1299 :       return FoldSymbolOffset(Opcode, VT, GA, Cst1);
    4203             : 
    4204             :   // For vectors extract each constant element into Inputs so we can constant
    4205             :   // fold them individually.
    4206             :   BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
    4207             :   BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
    4208     3538331 :   if (!BV1 || !BV2)
    4209     3535319 :     return SDValue();
    4210             : 
    4211             :   assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
    4212             : 
    4213        3012 :   EVT SVT = VT.getScalarType();
    4214        3012 :   EVT LegalSVT = SVT;
    4215        3012 :   if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
    4216        1540 :     LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
    4217         770 :     if (LegalSVT.bitsLT(SVT))
    4218           0 :       return SDValue();
    4219             :   }
    4220             :   SmallVector<SDValue, 4> Outputs;
    4221       24462 :   for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
    4222       21696 :     SDValue V1 = BV1->getOperand(I);
    4223       21696 :     SDValue V2 = BV2->getOperand(I);
    4224             : 
    4225       10848 :     if (SVT.isInteger()) {
    4226       19794 :         if (V1->getValueType(0).bitsGT(SVT))
    4227         353 :           V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
    4228       19794 :         if (V2->getValueType(0).bitsGT(SVT))
    4229         334 :           V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
    4230             :     }
    4231             : 
    4232       21796 :     if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
    4233           0 :       return SDValue();
    4234             : 
    4235             :     // Fold one vector element.
    4236       10848 :     SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
    4237       10898 :     if (LegalSVT != SVT)
    4238         299 :       ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
    4239             : 
    4240             :     // Scalar folding only succeeded if the result is a constant or UNDEF.
    4241       32529 :     if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
    4242             :         ScalarResult.getOpcode() != ISD::ConstantFP)
    4243        1629 :       return SDValue();
    4244        9219 :     Outputs.push_back(ScalarResult);
    4245             :   }
    4246             : 
    4247             :   assert(VT.getVectorNumElements() == Outputs.size() &&
    4248             :          "Vector size mismatch!");
    4249             : 
    4250             :   // We may have a vector type but a scalar result. Create a splat.
    4251        2766 :   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
    4252             : 
    4253             :   // Build a big vector out of the scalar elements we generated.
    4254        2766 :   return getBuildVector(VT, SDLoc(), Outputs);
    4255             : }
    4256             : 
    4257             : // TODO: Merge with FoldConstantArithmetic
    4258      795637 : SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
    4259             :                                                    const SDLoc &DL, EVT VT,
    4260             :                                                    ArrayRef<SDValue> Ops,
    4261             :                                                    const SDNodeFlags Flags) {
    4262             :   // If the opcode is a target-specific ISD node, there's nothing we can
    4263             :   // do here and the operand rules may not line up with the below, so
    4264             :   // bail early.
    4265      795637 :   if (Opcode >= ISD::BUILTIN_OP_END)
    4266           0 :     return SDValue();
    4267             : 
    4268      795637 :   if (isUndef(Opcode, Ops))
    4269           1 :     return getUNDEF(VT);
    4270             : 
    4271             :   // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
    4272      795636 :   if (!VT.isVector())
    4273      165254 :     return SDValue();
    4274             : 
    4275      630382 :   unsigned NumElts = VT.getVectorNumElements();
    4276             : 
    4277        3070 :   auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
    4278       15244 :     return !Op.getValueType().isVector() ||
    4279        6034 :            Op.getValueType().getVectorNumElements() == NumElts;
    4280        3070 :   };
    4281             : 
    4282      634886 :   auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
    4283             :     BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
    4284     1269584 :     return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
    4285        8347 :            (BV && BV->isConstant());
    4286             :   };
    4287             : 
    4288             :   // All operands must be vector types with the same number of elements as
    4289             :   // the result type and must be either UNDEF or a build vector of constant
    4290             :   // or UNDEF scalars.
    4291      632771 :   if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
    4292             :       !llvm::all_of(Ops, IsScalarOrSameVectorSize))
    4293      627993 :     return SDValue();
    4294             : 
    4295             :   // If we are comparing vectors, then the result needs to be a i1 boolean
    4296             :   // that is then sign-extended back to the legal result type.
    4297        4672 :   EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
    4298             : 
    4299             :   // Find legal integer scalar type for constant promotion and
    4300             :   // ensure that its scalar size is at least as large as source.
    4301        2389 :   EVT LegalSVT = VT.getScalarType();
    4302        2389 :   if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
    4303         774 :     LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
    4304         387 :     if (LegalSVT.bitsLT(VT.getScalarType()))
    4305           0 :       return SDValue();
    4306             :   }
    4307             : 
    4308             :   // Constant fold each scalar lane separately.
    4309             :   SmallVector<SDValue, 4> ScalarResults;
    4310       41459 :   for (unsigned i = 0; i != NumElts; i++) {
    4311             :     SmallVector<SDValue, 4> ScalarOps;
    4312       67192 :     for (SDValue Op : Ops) {
    4313       47544 :       EVT InSVT = Op.getValueType().getScalarType();
    4314             :       BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
    4315         478 :       if (!InBV) {
    4316             :         // We've checked that this is UNDEF or a constant of some kind.
    4317         478 :         if (Op.isUndef())
    4318         163 :           ScalarOps.push_back(getUNDEF(InSVT));
    4319             :         else
    4320         315 :           ScalarOps.push_back(Op);
    4321         478 :         continue;
    4322             :       }
    4323             : 
    4324       46588 :       SDValue ScalarOp = InBV->getOperand(i);
    4325       46588 :       EVT ScalarVT = ScalarOp.getValueType();
    4326             : 
    4327             :       // Build vector (integer) scalar operands may need implicit
    4328             :       // truncation - do this before constant folding.
    4329       23294 :       if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
    4330         202 :         ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
    4331             : 
    4332       23294 :       ScalarOps.push_back(ScalarOp);
    4333             :     }
    4334             : 
    4335             :     // Constant fold the scalar operands.
    4336       19648 :     SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
    4337             : 
    4338             :     // Legalize the (integer) scalar constant if necessary.
    4339       19664 :     if (LegalSVT != SVT)
    4340          78 :       ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
    4341             : 
    4342             :     // Scalar folding only succeeded if the result is a constant or UNDEF.
    4343       58915 :     if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
    4344             :         ScalarResult.getOpcode() != ISD::ConstantFP)
    4345         113 :       return SDValue();
    4346       19535 :     ScalarResults.push_back(ScalarResult);
    4347             :   }
    4348             : 
    4349        2276 :   SDValue V = getBuildVector(VT, DL, ScalarResults);
    4350             :   NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
    4351        2276 :   return V;
    4352             : }
    4353             : 
    4354     5350485 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4355             :                               SDValue N1, SDValue N2, const SDNodeFlags Flags) {
    4356             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    4357             :   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
    4358             :   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
    4359             :   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
    4360             : 
    4361             :   // Canonicalize constant to RHS if commutative.
    4362     5350485 :   if (TLI->isCommutativeBinOp(Opcode)) {
    4363     2470037 :     if (N1C && !N2C) {
    4364             :       std::swap(N1C, N2C);
    4365             :       std::swap(N1, N2);
    4366     2457653 :     } else if (N1CFP && !N2CFP) {
    4367             :       std::swap(N1CFP, N2CFP);
    4368             :       std::swap(N1, N2);
    4369             :     }
    4370             :   }
    4371             : 
    4372     5350485 :   switch (Opcode) {
    4373             :   default: break;
    4374     1185723 :   case ISD::TokenFactor:
    4375             :     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
    4376             :            N2.getValueType() == MVT::Other && "Invalid token factor!");
    4377             :     // Fold trivial token factors.
    4378     1185723 :     if (N1.getOpcode() == ISD::EntryToken) return N2;
    4379     1183946 :     if (N2.getOpcode() == ISD::EntryToken) return N1;
    4380         765 :     if (N1 == N2) return N1;
    4381             :     break;
    4382       32769 :   case ISD::CONCAT_VECTORS: {
    4383             :     // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
    4384       32769 :     SDValue Ops[] = {N1, N2};
    4385       32769 :     if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
    4386         747 :       return V;
    4387       32022 :     break;
    4388             :   }
    4389      185455 :   case ISD::AND:
    4390             :     assert(VT.isInteger() && "This operator does not apply to FP types!");
    4391             :     assert(N1.getValueType() == N2.getValueType() &&
    4392             :            N1.getValueType() == VT && "Binary operator types must match!");
    4393             :     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
    4394             :     // worth handling here.
    4395      341084 :     if (N2C && N2C->isNullValue())
    4396        2733 :       return N2;
    4397      488514 :     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
    4398        1943 :       return N1;
    4399             :     break;
    4400     2299110 :   case ISD::OR:
    4401             :   case ISD::XOR:
    4402             :   case ISD::ADD:
    4403             :   case ISD::SUB:
    4404             :     assert(VT.isInteger() && "This operator does not apply to FP types!");
    4405             :     assert(N1.getValueType() == N2.getValueType() &&
    4406             :            N1.getValueType() == VT && "Binary operator types must match!");
    4407             :     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
    4408             :     // it's worth handling here.
    4409     4085435 :     if (N2C && N2C->isNullValue())
    4410      717606 :       return N1;
    4411             :     break;
    4412             :   case ISD::UDIV:
    4413             :   case ISD::UREM:
    4414             :   case ISD::MULHU:
    4415             :   case ISD::MULHS:
    4416             :   case ISD::MUL:
    4417             :   case ISD::SDIV:
    4418             :   case ISD::SREM:
    4419             :   case ISD::SMIN:
    4420             :   case ISD::SMAX:
    4421             :   case ISD::UMIN:
    4422             :   case ISD::UMAX:
    4423             :     assert(VT.isInteger() && "This operator does not apply to FP types!");
    4424             :     assert(N1.getValueType() == N2.getValueType() &&
    4425             :            N1.getValueType() == VT && "Binary operator types must match!");
    4426             :     break;
    4427       37094 :   case ISD::FADD:
    4428             :   case ISD::FSUB:
    4429             :   case ISD::FMUL:
    4430             :   case ISD::FDIV:
    4431             :   case ISD::FREM:
    4432       37094 :     if (getTarget().Options.UnsafeFPMath) {
    4433        5833 :       if (Opcode == ISD::FADD) {
    4434             :         // x+0 --> x
    4435        1963 :         if (N2CFP && N2CFP->getValueAPF().isZero())
    4436          16 :           return N1;
    4437        4135 :       } else if (Opcode == ISD::FSUB) {
    4438             :         // x-0 --> x
    4439         807 :         if (N2CFP && N2CFP->getValueAPF().isZero())
    4440           0 :           return N1;
    4441        3338 :       } else if (Opcode == ISD::FMUL) {
    4442             :         // x*0 --> 0
    4443        3340 :         if (N2CFP && N2CFP->isZero())
    4444           6 :           return N2;
    4445             :         // x*1 --> x
    4446        3814 :         if (N2CFP && N2CFP->isExactlyValue(1.0))
    4447          61 :           return N1;
    4448             :       }
    4449             :     }
    4450             :     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
    4451             :     assert(N1.getValueType() == N2.getValueType() &&
    4452             :            N1.getValueType() == VT && "Binary operator types must match!");
    4453             :     break;
    4454             :   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
    4455             :     assert(N1.getValueType() == VT &&
    4456             :            N1.getValueType().isFloatingPoint() &&
    4457             :            N2.getValueType().isFloatingPoint() &&
    4458             :            "Invalid FCOPYSIGN!");
    4459             :     break;
    4460             :   case ISD::SHL:
    4461             :   case ISD::SRA:
    4462             :   case ISD::SRL:
    4463             :   case ISD::ROTL:
    4464             :   case ISD::ROTR:
    4465             :     assert(VT == N1.getValueType() &&
    4466             :            "Shift operators return type must be the same as their first arg");
    4467             :     assert(VT.isInteger() && N2.getValueType().isInteger() &&
    4468             :            "Shifts only work on integers");
    4469             :     assert((!VT.isVector() || VT == N2.getValueType()) &&
    4470             :            "Vector shift amounts must be in the same as their first arg");
    4471             :     // Verify that the shift amount VT is bit enough to hold valid shift
    4472             :     // amounts.  This catches things like trying to shift an i1024 value by an
    4473             :     // i8, which is easy to fall into in generic code that uses
    4474             :     // TLI.getShiftAmount().
    4475             :     assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) &&
    4476             :            "Invalid use of small shift amount with oversized value!");
    4477             : 
    4478             :     // Always fold shifts of i1 values so the code generator doesn't need to
    4479             :     // handle them.  Since we know the size of the shift has to be less than the
    4480             :     // size of the value, the shift/rotate count is guaranteed to be zero.
    4481             :     if (VT == MVT::i1)
    4482          45 :       return N1;
    4483     1092059 :     if (N2C && N2C->isNullValue())
    4484       10718 :       return N1;
    4485             :     break;
    4486             :   case ISD::FP_ROUND_INREG: {
    4487             :     EVT EVT = cast<VTSDNode>(N2)->getVT();
    4488             :     assert(VT == N1.getValueType() && "Not an inreg round!");
    4489             :     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
    4490             :            "Cannot FP_ROUND_INREG integer types");
    4491             :     assert(EVT.isVector() == VT.isVector() &&
    4492             :            "FP_ROUND_INREG type should be vector iff the operand "
    4493             :            "type is vector!");
    4494             :     assert((!EVT.isVector() ||
    4495             :             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
    4496             :            "Vector element counts must match in FP_ROUND_INREG");
    4497             :     assert(EVT.bitsLE(VT) && "Not rounding down!");
    4498             :     (void)EVT;
    4499           8 :     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
    4500             :     break;
    4501             :   }
    4502        5965 :   case ISD::FP_ROUND:
    4503             :     assert(VT.isFloatingPoint() &&
    4504             :            N1.getValueType().isFloatingPoint() &&
    4505             :            VT.bitsLE(N1.getValueType()) &&
    4506             :            N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
    4507             :            "Invalid FP_ROUND!");
    4508        5978 :     if (N1.getValueType() == VT) return N1;  // noop conversion.
    4509             :     break;
    4510             :   case ISD::AssertSext:
    4511             :   case ISD::AssertZext: {
    4512             :     EVT EVT = cast<VTSDNode>(N2)->getVT();
    4513             :     assert(VT == N1.getValueType() && "Not an inreg extend!");
    4514             :     assert(VT.isInteger() && EVT.isInteger() &&
    4515             :            "Cannot *_EXTEND_INREG FP types");
    4516             :     assert(!EVT.isVector() &&
    4517             :            "AssertSExt/AssertZExt type should be the vector element type "
    4518             :            "rather than the vector type!");
    4519             :     assert(EVT.bitsLE(VT) && "Not extending!");
    4520         378 :     if (VT == EVT) return N1; // noop assertion.
    4521             :     break;
    4522             :   }
    4523             :   case ISD::SIGN_EXTEND_INREG: {
    4524       18007 :     EVT EVT = cast<VTSDNode>(N2)->getVT();
    4525             :     assert(VT == N1.getValueType() && "Not an inreg extend!");
    4526             :     assert(VT.isInteger() && EVT.isInteger() &&
    4527             :            "Cannot *_EXTEND_INREG FP types");
    4528             :     assert(EVT.isVector() == VT.isVector() &&
    4529             :            "SIGN_EXTEND_INREG type should be vector iff the operand "
    4530             :            "type is vector!");
    4531             :     assert((!EVT.isVector() ||
    4532             :             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
    4533             :            "Vector element counts must match in SIGN_EXTEND_INREG");
    4534             :     assert(EVT.bitsLE(VT) && "Not extending!");
    4535       18379 :     if (EVT == VT) return N1;  // Not actually extending
    4536             : 
    4537         216 :     auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
    4538         216 :       unsigned FromBits = EVT.getScalarSizeInBits();
    4539         216 :       Val <<= Val.getBitWidth() - FromBits;
    4540         216 :       Val.ashrInPlace(Val.getBitWidth() - FromBits);
    4541         216 :       return getConstant(Val, DL, ConstantVT);
    4542       17917 :     };
    4543             : 
    4544       17917 :     if (N1C) {
    4545         186 :       const APInt &Val = N1C->getAPIntValue();
    4546         372 :       return SignExtendInReg(Val, VT);
    4547             :     }
    4548       17731 :     if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
    4549             :       SmallVector<SDValue, 8> Ops;
    4550          18 :       llvm::EVT OpVT = N1.getOperand(0).getValueType();
    4551          41 :       for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
    4552          64 :         SDValue Op = N1.getOperand(i);
    4553          34 :         if (Op.isUndef()) {
    4554           2 :           Ops.push_back(getUNDEF(OpVT));
    4555           2 :           continue;
    4556             :         }
    4557             :         ConstantSDNode *C = cast<ConstantSDNode>(Op);
    4558          30 :         APInt Val = C->getAPIntValue();
    4559          60 :         Ops.push_back(SignExtendInReg(Val, OpVT));
    4560             :       }
    4561           9 :       return getBuildVector(VT, DL, Ops);
    4562             :     }
    4563       17722 :     break;
    4564             :   }
    4565      288803 :   case ISD::EXTRACT_VECTOR_ELT:
    4566             :     assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
    4567             :            "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
    4568             :              element type of the vector.");
    4569             : 
    4570             :     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
    4571      288803 :     if (N1.isUndef())
    4572       14764 :       return getUNDEF(VT);
    4573             : 
    4574             :     // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
    4575      815021 :     if (N2C && N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
    4576          84 :       return getUNDEF(VT);
    4577             : 
    4578             :     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
    4579             :     // expanding copies of large vectors from registers.
    4580      270365 :     if (N2C &&
    4581      276992 :         N1.getOpcode() == ISD::CONCAT_VECTORS &&
    4582             :         N1.getNumOperands() > 0) {
    4583             :       unsigned Factor =
    4584        6074 :         N1.getOperand(0).getValueType().getVectorNumElements();
    4585             :       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
    4586        6074 :                      N1.getOperand(N2C->getZExtValue() / Factor),
    4587        3037 :                      getConstant(N2C->getZExtValue() % Factor, DL,
    4588        6074 :                                  N2.getValueType()));
    4589             :     }
    4590             : 
    4591             :     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
    4592             :     // expanding large vector constants.
    4593      538246 :     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
    4594      201460 :       SDValue Elt = N1.getOperand(N2C->getZExtValue());
    4595             : 
    4596      100770 :       if (VT != Elt.getValueType())
    4597             :         // If the vector element type is not legal, the BUILD_VECTOR operands
    4598             :         // are promoted and implicitly truncated, and the result implicitly
    4599             :         // extended. Make that explicit here.
    4600         311 :         Elt = getAnyExtOrTrunc(Elt, DL, VT);
    4601             : 
    4602      100730 :       return Elt;
    4603             :     }
    4604             : 
    4605             :     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
    4606             :     // operations are lowered to scalars.
    4607      170188 :     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
    4608             :       // If the indices are the same, return the inserted element else
    4609             :       // if the indices are known different, extract the element from
    4610             :       // the original vector.
    4611        4606 :       SDValue N1Op2 = N1.getOperand(2);
    4612             :       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
    4613             : 
    4614        3460 :       if (N1Op2C && N2C) {
    4615       10344 :         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
    4616         542 :           if (VT == N1.getOperand(1).getValueType())
    4617         517 :             return N1.getOperand(1);
    4618             :           else
    4619          25 :             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
    4620             :         }
    4621             : 
    4622        2906 :         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
    4623             :       }
    4624             :     }
    4625             : 
    4626             :     // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
    4627             :     // when vector types are scalarized and v1iX is legal.
    4628             :     // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx)
    4629      180647 :     if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
    4630      180617 :         N1.getValueType().getVectorNumElements() == 1) {
    4631             :       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
    4632          30 :                      N1.getOperand(1));
    4633             :     }
    4634             :     break;
    4635       31766 :   case ISD::EXTRACT_ELEMENT:
    4636             :     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
    4637             :     assert(!N1.getValueType().isVector() && !VT.isVector() &&
    4638             :            (N1.getValueType().isInteger() == VT.isInteger()) &&
    4639             :            N1.getValueType() != VT &&
    4640             :            "Wrong types for EXTRACT_ELEMENT!");
    4641             : 
    4642             :     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
    4643             :     // 64-bit integers into 32-bit parts.  Instead of building the extract of
    4644             :     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
    4645       31766 :     if (N1.getOpcode() == ISD::BUILD_PAIR)
    4646       11648 :       return N1.getOperand(N2C->getZExtValue());
    4647             : 
    4648             :     // EXTRACT_ELEMENT of a constant int is also very common.
    4649       25942 :     if (N1C) {
    4650        5394 :       unsigned ElementSize = VT.getSizeInBits();
    4651       10788 :       unsigned Shift = ElementSize * N2C->getZExtValue();
    4652       10788 :       APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
    4653       10788 :       return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
    4654       20548 :     }
    4655             :     break;
    4656       90560 :   case ISD::EXTRACT_SUBVECTOR:
    4657      180507 :     if (VT.isSimple() && N1.getValueType().isSimple()) {
    4658             :       assert(VT.isVector() && N1.getValueType().isVector() &&
    4659             :              "Extract subvector VTs must be a vectors!");
    4660             :       assert(VT.getVectorElementType() ==
    4661             :              N1.getValueType().getVectorElementType() &&
    4662             :              "Extract subvector VTs must have the same element type!");
    4663             :       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
    4664             :              "Extract subvector must be from larger vector to smaller vector!");
    4665             : 
    4666             :       if (N2C) {
    4667             :         assert((VT.getVectorNumElements() + N2C->getZExtValue()
    4668             :                 <= N1.getValueType().getVectorNumElements())
    4669             :                && "Extract subvector overflow!");
    4670             :       }
    4671             : 
    4672             :       // Trivial extraction.
    4673       86328 :       if (VT.getSimpleVT() == N1.getSimpleValueType())
    4674       14193 :         return N1;
    4675             : 
    4676             :       // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
    4677       72135 :       if (N1.isUndef())
    4678         760 :         return getUNDEF(VT);
    4679             : 
    4680             :       // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
    4681             :       // the concat have the same type as the extract.
    4682       75400 :       if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
    4683       71375 :           N1.getNumOperands() > 0 &&
    4684        4025 :           VT == N1.getOperand(0).getValueType()) {
    4685        3469 :         unsigned Factor = VT.getVectorNumElements();
    4686       10407 :         return N1.getOperand(N2C->getZExtValue() / Factor);
    4687             :       }
    4688             : 
    4689             :       // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
    4690             :       // during shuffle legalization.
    4691       67906 :       if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
    4692         626 :           VT == N1.getOperand(1).getValueType())
    4693         274 :         return N1.getOperand(1);
    4694             :     }
    4695             :     break;
    4696             :   }
    4697             : 
    4698             :   // Perform trivial constant folding.
    4699     4435785 :   if (SDValue SV =
    4700     4435785 :           FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
    4701      190974 :     return SV;
    4702             : 
    4703             :   // Constant fold FP operations.
    4704     4244811 :   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
    4705     4244811 :   if (N1CFP) {
    4706        3580 :     if (N2CFP) {
    4707        4482 :       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
    4708             :       APFloat::opStatus s;
    4709        2241 :       switch (Opcode) {
    4710         311 :       case ISD::FADD:
    4711         311 :         s = V1.add(V2, APFloat::rmNearestTiesToEven);
    4712         311 :         if (!HasFPExceptions || s != APFloat::opInvalidOp)
    4713         311 :           return getConstantFP(V1, DL, VT);
    4714             :         break;
    4715          27 :       case ISD::FSUB:
    4716          27 :         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
    4717          27 :         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
    4718          27 :           return getConstantFP(V1, DL, VT);
    4719             :         break;
    4720         278 :       case ISD::FMUL:
    4721         278 :         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
    4722         278 :         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
    4723         278 :           return getConstantFP(V1, DL, VT);
    4724             :         break;
    4725          43 :       case ISD::FDIV:
    4726          43 :         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
    4727          43 :         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
    4728             :                                  s!=APFloat::opDivByZero)) {
    4729          20 :           return getConstantFP(V1, DL, VT);
    4730             :         }
    4731             :         break;
    4732          25 :       case ISD::FREM :
    4733          25 :         s = V1.mod(V2);
    4734          25 :         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
    4735             :                                  s!=APFloat::opDivByZero)) {
    4736          11 :           return getConstantFP(V1, DL, VT);
    4737             :         }
    4738             :         break;
    4739           4 :       case ISD::FCOPYSIGN:
    4740           4 :         V1.copySign(V2);
    4741           4 :         return getConstantFP(V1, DL, VT);
    4742             :       default: break;
    4743             :       }
    4744             :     }
    4745             : 
    4746        2929 :     if (Opcode == ISD::FP_ROUND) {
    4747           4 :       APFloat V = N1CFP->getValueAPF();    // make copy
    4748             :       bool ignored;
    4749             :       // This can return overflow, underflow, or inexact; we don't care.
    4750             :       // FIXME need to be more flexible about rounding mode.
    4751           4 :       (void)V.convert(EVTToAPFloatSemantics(VT),
    4752             :                       APFloat::rmNearestTiesToEven, &ignored);
    4753           4 :       return getConstantFP(V, DL, VT);
    4754             :     }
    4755             :   }
    4756             : 
    4757             :   // Canonicalize an UNDEF to the RHS, even over a constant.
    4758     4244156 :   if (N1.isUndef()) {
    4759        4567 :     if (TLI->isCommutativeBinOp(Opcode)) {
    4760             :       std::swap(N1, N2);
    4761             :     } else {
    4762        1621 :       switch (Opcode) {
    4763          46 :       case ISD::FP_ROUND_INREG:
    4764             :       case ISD::SIGN_EXTEND_INREG:
    4765             :       case ISD::SUB:
    4766             :       case ISD::FSUB:
    4767             :       case ISD::FDIV:
    4768             :       case ISD::FREM:
    4769          46 :         return getUNDEF(VT);     // fold op(undef, arg2) -> undef
    4770         249 :       case ISD::UDIV:
    4771             :       case ISD::SDIV:
    4772             :       case ISD::UREM:
    4773             :       case ISD::SREM:
    4774             :       case ISD::SRA:
    4775             :       case ISD::SRL:
    4776             :       case ISD::SHL:
    4777         249 :         return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
    4778             :       }
    4779             :     }
    4780             :   }
    4781             : 
    4782             :   // Fold a bunch of operators when the RHS is undef.
    4783     4243861 :   if (N2.isUndef()) {
    4784       12388 :     switch (Opcode) {
    4785         495 :     case ISD::XOR:
    4786         495 :       if (N1.isUndef())
    4787             :         // Handle undef ^ undef -> 0 special case. This is a common
    4788             :         // idiom (misuse).
    4789           6 :         return getConstant(0, DL, VT);
    4790             :       LLVM_FALLTHROUGH;
    4791             :     case ISD::ADD:
    4792             :     case ISD::ADDC:
    4793             :     case ISD::ADDE:
    4794             :     case ISD::SUB:
    4795             :     case ISD::UDIV:
    4796             :     case ISD::SDIV:
    4797             :     case ISD::UREM:
    4798             :     case ISD::SREM:
    4799             :     case ISD::SRA:
    4800             :     case ISD::SRL:
    4801             :     case ISD::SHL:
    4802        2187 :       return getUNDEF(VT);       // fold op(arg1, undef) -> undef
    4803        1347 :     case ISD::FADD:
    4804             :     case ISD::FSUB:
    4805             :     case ISD::FMUL:
    4806             :     case ISD::FDIV:
    4807             :     case ISD::FREM:
    4808        1347 :       if (getTarget().Options.UnsafeFPMath)
    4809           1 :         return N2;
    4810             :       break;
    4811         169 :     case ISD::MUL:
    4812             :     case ISD::AND:
    4813         169 :       return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
    4814          86 :     case ISD::OR:
    4815          86 :       return getAllOnesConstant(DL, VT);
    4816             :     }
    4817             :   }
    4818             : 
    4819             :   // Memoize this node if possible.
    4820             :   SDNode *N;
    4821     4241412 :   SDVTList VTs = getVTList(VT);
    4822     4241412 :   SDValue Ops[] = {N1, N2};
    4823     4241412 :   if (VT != MVT::Glue) {
    4824             :     FoldingSetNodeID ID;
    4825     4237958 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    4826     4237958 :     void *IP = nullptr;
    4827     4237958 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
    4828      444732 :       E->intersectFlagsWith(Flags);
    4829      444732 :       return SDValue(E, 0);
    4830             :     }
    4831             : 
    4832     7586452 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4833             :     N->setFlags(Flags);
    4834     3793226 :     createOperands(N, Ops);
    4835     3793226 :     CSEMap.InsertNode(N, IP);
    4836             :   } else {
    4837        6908 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4838        3454 :     createOperands(N, Ops);
    4839             :   }
    4840             : 
    4841     3796680 :   InsertNode(N);
    4842             :   SDValue V = SDValue(N, 0);
    4843             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    4844     3796680 :   return V;
    4845             : }
    4846             : 
    4847      819776 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4848             :                               SDValue N1, SDValue N2, SDValue N3) {
    4849             :   // Perform various simplifications.
    4850      819776 :   switch (Opcode) {
    4851             :   case ISD::FMA: {
    4852             :     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
    4853             :     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
    4854             :     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
    4855        3669 :     if (N1CFP && N2CFP && N3CFP) {
    4856           8 :       APFloat  V1 = N1CFP->getValueAPF();
    4857           8 :       const APFloat &V2 = N2CFP->getValueAPF();
    4858           8 :       const APFloat &V3 = N3CFP->getValueAPF();
    4859             :       APFloat::opStatus s =
    4860           8 :         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
    4861           8 :       if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
    4862           8 :         return getConstantFP(V1, DL, VT);
    4863             :     }
    4864             :     break;
    4865             :   }
    4866          10 :   case ISD::CONCAT_VECTORS: {
    4867             :     // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
    4868          10 :     SDValue Ops[] = {N1, N2, N3};
    4869          10 :     if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
    4870           2 :       return V;
    4871           8 :     break;
    4872             :   }
    4873             :   case ISD::SETCC: {
    4874             :     // Use FoldSetCC to simplify SETCC's.
    4875      196144 :     if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
    4876         997 :       return V;
    4877             :     // Vector constant folding.
    4878      195147 :     SDValue Ops[] = {N1, N2, N3};
    4879      195147 :     if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
    4880             :       NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
    4881          57 :       return V;
    4882             :     }
    4883      195090 :     break;
    4884             :   }
    4885             :   case ISD::SELECT:
    4886             :     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
    4887         776 :      if (N1C->getZExtValue())
    4888         132 :        return N2;             // select true, X, Y -> X
    4889         256 :      return N3;             // select false, X, Y -> Y
    4890             :     }
    4891             : 
    4892          75 :     if (N2 == N3) return N2;   // select C, X, X -> X
    4893             :     break;
    4894           0 :   case ISD::VECTOR_SHUFFLE:
    4895           0 :     llvm_unreachable("should use getVectorShuffle constructor!");
    4896             :   case ISD::INSERT_VECTOR_ELT: {
    4897             :     ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
    4898             :     // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF
    4899       71572 :     if (N3C && N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
    4900           8 :       return getUNDEF(VT);
    4901             :     break;
    4902             :   }
    4903       18759 :   case ISD::INSERT_SUBVECTOR: {
    4904             :     SDValue Index = N3;
    4905       18759 :     if (VT.isSimple() && N1.getValueType().isSimple()
    4906       37518 :         && N2.getValueType().isSimple()) {
    4907             :       assert(VT.isVector() && N1.getValueType().isVector() &&
    4908             :              N2.getValueType().isVector() &&
    4909             :              "Insert subvector VTs must be a vectors");
    4910             :       assert(VT == N1.getValueType() &&
    4911             :              "Dest and insert subvector source types must match!");
    4912             :       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
    4913             :              "Insert subvector must be from smaller vector to larger vector!");
    4914             :       if (isa<ConstantSDNode>(Index)) {
    4915             :         assert((N2.getValueType().getVectorNumElements() +
    4916             :                 cast<ConstantSDNode>(Index)->getZExtValue()
    4917             :                 <= VT.getVectorNumElements())
    4918             :                && "Insert subvector overflow!");
    4919             :       }
    4920             : 
    4921             :       // Trivial insertion.
    4922       18759 :       if (VT.getSimpleVT() == N2.getSimpleValueType())
    4923          51 :         return N2;
    4924             :     }
    4925             :     break;
    4926             :   }
    4927           0 :   case ISD::BITCAST:
    4928             :     // Fold bit_convert nodes from a type to themselves.
    4929           0 :     if (N1.getValueType() == VT)
    4930           0 :       return N1;
    4931             :     break;
    4932             :   }
    4933             : 
    4934             :   // Memoize node if it doesn't produce a flag.
    4935             :   SDNode *N;
    4936      818190 :   SDVTList VTs = getVTList(VT);
    4937      818190 :   SDValue Ops[] = {N1, N2, N3};
    4938             :   if (VT != MVT::Glue) {
    4939             :     FoldingSetNodeID ID;
    4940      816172 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    4941      816172 :     void *IP = nullptr;
    4942      816172 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    4943       98893 :       return SDValue(E, 0);
    4944             : 
    4945     1434558 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4946      717279 :     createOperands(N, Ops);
    4947      717279 :     CSEMap.InsertNode(N, IP);
    4948             :   } else {
    4949        4036 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4950        2018 :     createOperands(N, Ops);
    4951             :   }
    4952             : 
    4953      719297 :   InsertNode(N);
    4954             :   SDValue V = SDValue(N, 0);
    4955             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    4956      719297 :   return V;
    4957             : }
    4958             : 
    4959       38326 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4960             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
    4961       38326 :   SDValue Ops[] = { N1, N2, N3, N4 };
    4962       38326 :   return getNode(Opcode, DL, VT, Ops);
    4963             : }
    4964             : 
    4965       38175 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4966             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
    4967             :                               SDValue N5) {
    4968       38175 :   SDValue Ops[] = { N1, N2, N3, N4, N5 };
    4969       38175 :   return getNode(Opcode, DL, VT, Ops);
    4970             : }
    4971             : 
    4972             : /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
    4973             : /// the incoming stack arguments to be loaded from the stack.
    4974         154 : SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
    4975             :   SmallVector<SDValue, 8> ArgChains;
    4976             : 
    4977             :   // Include the original chain at the beginning of the list. When this is
    4978             :   // used by target LowerCall hooks, this helps legalize find the
    4979             :   // CALLSEQ_BEGIN node.
    4980         154 :   ArgChains.push_back(Chain);
    4981             : 
    4982             :   // Add a chain value for each stack argument.
    4983         154 :   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
    4984         738 :        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
    4985             :     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
    4986             :       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
    4987         102 :         if (FI->getIndex() < 0)
    4988         102 :           ArgChains.push_back(SDValue(L, 1));
    4989             : 
    4990             :   // Build a tokenfactor for all the chains.
    4991         462 :   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
    4992             : }
    4993             : 
    4994             : /// getMemsetValue - Vectorized representation of the memset value
    4995             : /// operand.
    4996       23383 : static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
    4997             :                               const SDLoc &dl) {
    4998             :   assert(!Value.isUndef());
    4999             : 
    5000             :   unsigned NumBits = VT.getScalarSizeInBits();
    5001             :   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
    5002             :     assert(C->getAPIntValue().getBitWidth() == 8);
    5003       46698 :     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
    5004       23349 :     if (VT.isInteger())
    5005       21106 :       return DAG.getConstant(Val, dl, VT);
    5006        4486 :     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
    5007        2243 :                              VT);
    5008             :   }
    5009             : 
    5010             :   assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
    5011          34 :   EVT IntVT = VT.getScalarType();
    5012          34 :   if (!IntVT.isInteger())
    5013           1 :     IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
    5014             : 
    5015          34 :   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
    5016          34 :   if (NumBits > 8) {
    5017             :     // Use a multiplication with 0x010101... to extend the input to the
    5018             :     // required length.
    5019          36 :     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
    5020          18 :     Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
    5021          36 :                         DAG.getConstant(Magic, dl, IntVT));
    5022             :   }
    5023             : 
    5024          34 :   if (VT != Value.getValueType() && !VT.isInteger())
    5025           1 :     Value = DAG.getBitcast(VT.getScalarType(), Value);
    5026          34 :   if (VT != Value.getValueType())
    5027          18 :     Value = DAG.getSplatBuildVector(VT, dl, Value);
    5028             : 
    5029          34 :   return Value;
    5030             : }
    5031             : 
    5032             : /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
    5033             : /// used when a memcpy is turned into a memset when the source is a constant
    5034             : /// string ptr.
    5035         332 : static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
    5036             :                                   const TargetLowering &TLI,
    5037             :                                   const ConstantDataArraySlice &Slice) {
    5038             :   // Handle vector with all elements zero.
    5039         332 :   if (Slice.Array == nullptr) {
    5040          45 :     if (VT.isInteger())
    5041          42 :       return DAG.getConstant(0, dl, VT);
    5042             :     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
    5043           3 :       return DAG.getConstantFP(0.0, dl, VT);
    5044           0 :     else if (VT.isVector()) {
    5045           0 :       unsigned NumElts = VT.getVectorNumElements();
    5046           0 :       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
    5047             :       return DAG.getNode(ISD::BITCAST, dl, VT,
    5048             :                          DAG.getConstant(0, dl,
    5049           0 :                                          EVT::getVectorVT(*DAG.getContext(),
    5050           0 :                                                           EltVT, NumElts)));
    5051             :     } else
    5052           0 :       llvm_unreachable("Expected type!");
    5053             :   }
    5054             : 
    5055             :   assert(!VT.isVector() && "Can't handle vector type here!");
    5056         287 :   unsigned NumVTBits = VT.getSizeInBits();
    5057         287 :   unsigned NumVTBytes = NumVTBits / 8;
    5058         574 :   unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
    5059             : 
    5060             :   APInt Val(NumVTBits, 0);
    5061         574 :   if (DAG.getDataLayout().isLittleEndian()) {
    5062        2303 :     for (unsigned i = 0; i != NumBytes; ++i)
    5063        2016 :       Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
    5064             :   } else {
    5065           0 :     for (unsigned i = 0; i != NumBytes; ++i)
    5066           0 :       Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
    5067             :   }
    5068             : 
    5069             :   // If the "cost" of materializing the integer immediate is less than the cost
    5070             :   // of a load, then it is cost effective to turn the load into the immediate.
    5071         287 :   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
    5072         287 :   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
    5073         272 :     return DAG.getConstant(Val, dl, VT);
    5074          15 :   return SDValue(nullptr, 0);
    5075             : }
    5076             : 
    5077       50316 : SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset,
    5078             :                                            const SDLoc &DL) {
    5079      100632 :   EVT VT = Base.getValueType();
    5080       50316 :   return getNode(ISD::ADD, DL, VT, Base, getConstant(Offset, DL, VT));
    5081             : }
    5082             : 
    5083             : /// Returns true if memcpy source is constant data.
    5084        1476 : static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
    5085             :   uint64_t SrcDelta = 0;
    5086             :   GlobalAddressSDNode *G = nullptr;
    5087        1476 :   if (Src.getOpcode() == ISD::GlobalAddress)
    5088             :     G = cast<GlobalAddressSDNode>(Src);
    5089          94 :   else if (Src.getOpcode() == ISD::ADD &&
    5090        1258 :            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
    5091          10 :            Src.getOperand(1).getOpcode() == ISD::Constant) {
    5092             :     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
    5093           9 :     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
    5094             :   }
    5095         331 :   if (!G)
    5096             :     return false;
    5097             : 
    5098         331 :   return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
    5099         662 :                                   SrcDelta + G->getOffset());
    5100             : }
    5101             : 
    5102             : /// Determines the optimal series of memory ops to replace the memset / memcpy.
    5103             : /// Return true if the number of memory ops is below the threshold (Limit).
    5104             : /// It returns the types of the sequence of memory ops to perform
    5105             : /// memset / memcpy by reference.
    5106       24466 : static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
    5107             :                                      unsigned Limit, uint64_t Size,
    5108             :                                      unsigned DstAlign, unsigned SrcAlign,
    5109             :                                      bool IsMemset,
    5110             :                                      bool ZeroMemset,
    5111             :                                      bool MemcpyStrSrc,
    5112             :                                      bool AllowOverlap,
    5113             :                                      unsigned DstAS, unsigned SrcAS,
    5114             :                                      SelectionDAG &DAG,
    5115             :                                      const TargetLowering &TLI) {
    5116             :   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
    5117             :          "Expecting memcpy / memset source to meet alignment requirement!");
    5118             :   // If 'SrcAlign' is zero, that means the memory operation does not need to
    5119             :   // load the value, i.e. memset or memcpy from constant string. Otherwise,
    5120             :   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
    5121             :   // is the specified alignment of the memory operation. If it is zero, that
    5122             :   // means it's possible to change the alignment of the destination.
    5123             :   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
    5124             :   // not need to be loaded.
    5125             :   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
    5126             :                                    IsMemset, ZeroMemset, MemcpyStrSrc,
    5127       24466 :                                    DAG.getMachineFunction());
    5128             : 
    5129             :   if (VT == MVT::Other) {
    5130             :     // Use the largest integer type whose alignment constraints are satisfied.
    5131             :     // We only need to check DstAlign here as SrcAlign is always greater or
    5132             :     // equal to DstAlign (or zero).
    5133         468 :     VT = MVT::i64;
    5134        1692 :     while (DstAlign && DstAlign < VT.getSizeInBits() / 8 &&
    5135         584 :            !TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign))
    5136         320 :       VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
    5137             :     assert(VT.isInteger());
    5138             : 
    5139             :     // Find the largest legal integer type.
    5140             :     MVT LVT = MVT::i64;
    5141         326 :     while (!TLI.isTypeLegal(LVT))
    5142         326 :       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
    5143             :     assert(LVT.isInteger());
    5144             : 
    5145             :     // If the type we've chosen is larger than the largest legal integer type
    5146             :     // then use that instead.
    5147         468 :     if (VT.bitsGT(LVT))
    5148          63 :       VT = LVT;
    5149             :   }
    5150             : 
    5151             :   unsigned NumMemOps = 0;
    5152      146012 :   while (Size != 0) {
    5153       62070 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5154       66592 :     while (VTSize > Size) {
    5155             :       // For now, only use non-vector load / store's for the left-over pieces.
    5156        2261 :       EVT NewVT = VT;
    5157             :       unsigned NewVTSize;
    5158             : 
    5159             :       bool Found = false;
    5160        2261 :       if (VT.isVector() || VT.isFloatingPoint()) {
    5161        1649 :         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
    5162         974 :         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
    5163         974 :             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
    5164             :           Found = true;
    5165             :         else if (NewVT == MVT::i64 &&
    5166         617 :                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
    5167        1293 :                  TLI.isSafeMemOpType(MVT::f64)) {
    5168             :           // i64 is usually not legal on 32-bit targets, but f64 may be.
    5169         616 :           NewVT = MVT::f64;
    5170             :           Found = true;
    5171             :         }
    5172             :       }
    5173             : 
    5174             :       if (!Found) {
    5175             :         do {
    5176         671 :           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
    5177             :           if (NewVT == MVT::i8)
    5178             :             break;
    5179         519 :         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
    5180             :       }
    5181        2261 :       NewVTSize = NewVT.getSizeInBits() / 8;
    5182             : 
    5183             :       // If the new VT cannot cover all of the remaining bits, then consider
    5184             :       // issuing a (or a pair of) unaligned and overlapping load / store.
    5185             :       // FIXME: Only does this for 64-bit or more since we don't have proper
    5186             :       // cost model for unaligned load / store.
    5187             :       bool Fast;
    5188        4285 :       if (NumMemOps && AllowOverlap &&
    5189        1864 :           VTSize >= 8 && NewVTSize < Size &&
    5190        2475 :           TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast) && Fast)
    5191          98 :         VTSize = Size;
    5192             :       else {
    5193        2163 :         VT = NewVT;
    5194             :         VTSize = NewVTSize;
    5195             :       }
    5196             :     }
    5197             : 
    5198       62070 :     if (++NumMemOps > Limit)
    5199             :       return false;
    5200             : 
    5201       60773 :     MemOps.push_back(VT);
    5202       60773 :     Size -= VTSize;
    5203             :   }
    5204             : 
    5205             :   return true;
    5206             : }
    5207             : 
    5208       24466 : static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
    5209             :   // On Darwin, -Os means optimize for size without hurting performance, so
    5210             :   // only really optimize for size when -Oz (MinSize) is used.
    5211       24466 :   if (MF.getTarget().getTargetTriple().isOSDarwin())
    5212         324 :     return MF.getFunction().optForMinSize();
    5213       24304 :   return MF.getFunction().optForSize();
    5214             : }
    5215             : 
    5216        1481 : static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
    5217             :                                        SDValue Chain, SDValue Dst, SDValue Src,
    5218             :                                        uint64_t Size, unsigned Align,
    5219             :                                        bool isVol, bool AlwaysInline,
    5220             :                                        MachinePointerInfo DstPtrInfo,
    5221             :                                        MachinePointerInfo SrcPtrInfo) {
    5222             :   // Turn a memcpy of undef to nop.
    5223        1481 :   if (Src.isUndef())
    5224           5 :     return Chain;
    5225             : 
    5226             :   // Expand memcpy to a series of load and store ops if the size operand falls
    5227             :   // below a certain threshold.
    5228             :   // TODO: In the AlwaysInline case, if the size is big then generate a loop
    5229             :   // rather than maybe a humongous number of loads and stores.
    5230        1476 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5231        1476 :   const DataLayout &DL = DAG.getDataLayout();
    5232        1476 :   LLVMContext &C = *DAG.getContext();
    5233             :   std::vector<EVT> MemOps;
    5234             :   bool DstAlignCanChange = false;
    5235        1476 :   MachineFunction &MF = DAG.getMachineFunction();
    5236        1476 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    5237        1476 :   bool OptSize = shouldLowerMemFuncForSize(MF);
    5238             :   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
    5239         262 :   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
    5240             :     DstAlignCanChange = true;
    5241        1476 :   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
    5242        1476 :   if (Align > SrcAlign)
    5243             :     SrcAlign = Align;
    5244             :   ConstantDataArraySlice Slice;
    5245        1476 :   bool CopyFromConstant = isMemSrcFromConstant(Src, Slice);
    5246        1476 :   bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
    5247        1476 :   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
    5248             : 
    5249        1476 :   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
    5250             :                                 (DstAlignCanChange ? 0 : Align),
    5251             :                                 (isZeroConstant ? 0 : SrcAlign),
    5252             :                                 false, false, CopyFromConstant, true,
    5253             :                                 DstPtrInfo.getAddrSpace(),
    5254             :                                 SrcPtrInfo.getAddrSpace(),
    5255             :                                 DAG, TLI))
    5256         276 :     return SDValue();
    5257             : 
    5258        1200 :   if (DstAlignCanChange) {
    5259         240 :     Type *Ty = MemOps[0].getTypeForEVT(C);
    5260         240 :     unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
    5261             : 
    5262             :     // Don't promote to an alignment that would require dynamic stack
    5263             :     // realignment.
    5264         240 :     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    5265         240 :     if (!TRI->needsStackRealignment(MF))
    5266         239 :       while (NewAlign > Align &&
    5267         155 :              DL.exceedsNaturalStackAlignment(NewAlign))
    5268           3 :           NewAlign /= 2;
    5269             : 
    5270         240 :     if (NewAlign > Align) {
    5271             :       // Give the stack frame object a larger alignment if needed.
    5272         314 :       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
    5273             :         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
    5274             :       Align = NewAlign;
    5275             :     }
    5276             :   }
    5277             : 
    5278             :   MachineMemOperand::Flags MMOFlags =
    5279        1200 :       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
    5280             :   SmallVector<SDValue, 8> OutChains;
    5281        2400 :   unsigned NumMemOps = MemOps.size();
    5282             :   uint64_t SrcOff = 0, DstOff = 0;
    5283        6132 :   for (unsigned i = 0; i != NumMemOps; ++i) {
    5284        4932 :     EVT VT = MemOps[i];
    5285        2466 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5286        2466 :     SDValue Value, Store;
    5287             : 
    5288        2466 :     if (VTSize > Size) {
    5289             :       // Issuing an unaligned load / store pair  that overlaps with the previous
    5290             :       // pair. Adjust the offset accordingly.
    5291             :       assert(i == NumMemOps-1 && i != 0);
    5292          94 :       SrcOff -= VTSize - Size;
    5293             :       DstOff -= VTSize - Size;
    5294             :     }
    5295             : 
    5296        2466 :     if (CopyFromConstant &&
    5297         733 :         (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
    5298             :       // It's unlikely a store of a vector immediate can be done in a single
    5299             :       // instruction. It would require a load from a constantpool first.
    5300             :       // We only handle zero vectors here.
    5301             :       // FIXME: Handle other cases where store of vector immediate is done in
    5302             :       // a single instruction.
    5303             :       ConstantDataArraySlice SubSlice;
    5304         332 :       if (SrcOff < Slice.Length) {
    5305         288 :         SubSlice = Slice;
    5306             :         SubSlice.move(SrcOff);
    5307             :       } else {
    5308             :         // This is an out-of-bounds access and hence UB. Pretend we read zero.
    5309          44 :         SubSlice.Array = nullptr;
    5310          44 :         SubSlice.Offset = 0;
    5311          44 :         SubSlice.Length = VTSize;
    5312             :       }
    5313         332 :       Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
    5314         332 :       if (Value.getNode())
    5315         317 :         Store = DAG.getStore(Chain, dl, Value,
    5316             :                              DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5317             :                              DstPtrInfo.getWithOffset(DstOff), Align,
    5318         634 :                              MMOFlags);
    5319             :     }
    5320             : 
    5321        2466 :     if (!Store.getNode()) {
    5322             :       // The type might not be legal for the target.  This should only happen
    5323             :       // if the type is smaller than a legal type, as on PPC, so the right
    5324             :       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
    5325             :       // to Load/Store if NVT==VT.
    5326             :       // FIXME does the case above also need this?
    5327        4298 :       EVT NVT = TLI.getTypeToTransformTo(C, VT);
    5328             :       assert(NVT.bitsGE(VT));
    5329             : 
    5330             :       bool isDereferenceable =
    5331        2149 :         SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
    5332             :       MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
    5333        2149 :       if (isDereferenceable)
    5334             :         SrcMMOFlags |= MachineMemOperand::MODereferenceable;
    5335             : 
    5336        2149 :       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
    5337             :                              DAG.getMemBasePlusOffset(Src, SrcOff, dl),
    5338             :                              SrcPtrInfo.getWithOffset(SrcOff), VT,
    5339        6447 :                              MinAlign(SrcAlign, SrcOff), SrcMMOFlags);
    5340        2149 :       OutChains.push_back(Value.getValue(1));
    5341        2149 :       Store = DAG.getTruncStore(
    5342             :           Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5343        4298 :           DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags);
    5344             :     }
    5345        2466 :     OutChains.push_back(Store);
    5346        2466 :     SrcOff += VTSize;
    5347             :     DstOff += VTSize;
    5348        2466 :     Size -= VTSize;
    5349             :   }
    5350             : 
    5351        1200 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    5352             : }
    5353             : 
    5354          83 : static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
    5355             :                                         SDValue Chain, SDValue Dst, SDValue Src,
    5356             :                                         uint64_t Size, unsigned Align,
    5357             :                                         bool isVol, bool AlwaysInline,
    5358             :                                         MachinePointerInfo DstPtrInfo,
    5359             :                                         MachinePointerInfo SrcPtrInfo) {
    5360             :   // Turn a memmove of undef to nop.
    5361         166 :   if (Src.isUndef())
    5362           0 :     return Chain;
    5363             : 
    5364             :   // Expand memmove to a series of load and store ops if the size operand falls
    5365             :   // below a certain threshold.
    5366          83 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5367          83 :   const DataLayout &DL = DAG.getDataLayout();
    5368          83 :   LLVMContext &C = *DAG.getContext();
    5369             :   std::vector<EVT> MemOps;
    5370             :   bool DstAlignCanChange = false;
    5371          83 :   MachineFunction &MF = DAG.getMachineFunction();
    5372          83 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    5373          83 :   bool OptSize = shouldLowerMemFuncForSize(MF);
    5374             :   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
    5375           1 :   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
    5376             :     DstAlignCanChange = true;
    5377          83 :   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
    5378          83 :   if (Align > SrcAlign)
    5379             :     SrcAlign = Align;
    5380          83 :   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
    5381             : 
    5382          83 :   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
    5383             :                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
    5384             :                                 false, false, false, false,
    5385             :                                 DstPtrInfo.getAddrSpace(),
    5386             :                                 SrcPtrInfo.getAddrSpace(),
    5387             :                                 DAG, TLI))
    5388          57 :     return SDValue();
    5389             : 
    5390          26 :   if (DstAlignCanChange) {
    5391           1 :     Type *Ty = MemOps[0].getTypeForEVT(C);
    5392           1 :     unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
    5393           1 :     if (NewAlign > Align) {
    5394             :       // Give the stack frame object a larger alignment if needed.
    5395           2 :       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
    5396             :         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
    5397             :       Align = NewAlign;
    5398             :     }
    5399             :   }
    5400             : 
    5401             :   MachineMemOperand::Flags MMOFlags =
    5402          26 :       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
    5403             :   uint64_t SrcOff = 0, DstOff = 0;
    5404             :   SmallVector<SDValue, 8> LoadValues;
    5405             :   SmallVector<SDValue, 8> LoadChains;
    5406             :   SmallVector<SDValue, 8> OutChains;
    5407          52 :   unsigned NumMemOps = MemOps.size();
    5408          78 :   for (unsigned i = 0; i < NumMemOps; i++) {
    5409          52 :     EVT VT = MemOps[i];
    5410          26 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5411          26 :     SDValue Value;
    5412             : 
    5413             :     bool isDereferenceable =
    5414          26 :       SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
    5415             :     MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
    5416          26 :     if (isDereferenceable)
    5417             :       SrcMMOFlags |= MachineMemOperand::MODereferenceable;
    5418             : 
    5419          26 :     Value =
    5420          52 :         DAG.getLoad(VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl),
    5421          52 :                     SrcPtrInfo.getWithOffset(SrcOff), SrcAlign, SrcMMOFlags);
    5422          26 :     LoadValues.push_back(Value);
    5423          26 :     LoadChains.push_back(Value.getValue(1));
    5424          26 :     SrcOff += VTSize;
    5425             :   }
    5426          26 :   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
    5427             :   OutChains.clear();
    5428          78 :   for (unsigned i = 0; i < NumMemOps; i++) {
    5429          52 :     EVT VT = MemOps[i];
    5430          26 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5431          26 :     SDValue Store;
    5432             : 
    5433          26 :     Store = DAG.getStore(Chain, dl, LoadValues[i],
    5434             :                          DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5435          52 :                          DstPtrInfo.getWithOffset(DstOff), Align, MMOFlags);
    5436          26 :     OutChains.push_back(Store);
    5437          26 :     DstOff += VTSize;
    5438             :   }
    5439             : 
    5440          26 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    5441             : }
    5442             : 
    5443             : /// \brief Lower the call to 'memset' intrinsic function into a series of store
    5444             : /// operations.
    5445             : ///
    5446             : /// \param DAG Selection DAG where lowered code is placed.
    5447             : /// \param dl Link to corresponding IR location.
    5448             : /// \param Chain Control flow dependency.
    5449             : /// \param Dst Pointer to destination memory location.
    5450             : /// \param Src Value of byte to write into the memory.
    5451             : /// \param Size Number of bytes to write.
    5452             : /// \param Align Alignment of the destination in bytes.
    5453             : /// \param isVol True if destination is volatile.
    5454             : /// \param DstPtrInfo IR information on the memory pointer.
    5455             : /// \returns New head in the control flow, if lowering was successful, empty
    5456             : /// SDValue otherwise.
    5457             : ///
    5458             : /// The function tries to replace 'llvm.memset' intrinsic with several store
    5459             : /// operations and value calculation code. This is usually profitable for small
    5460             : /// memory size.
    5461       22908 : static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
    5462             :                                SDValue Chain, SDValue Dst, SDValue Src,
    5463             :                                uint64_t Size, unsigned Align, bool isVol,
    5464             :                                MachinePointerInfo DstPtrInfo) {
    5465             :   // Turn a memset of undef to nop.
    5466       22908 :   if (Src.isUndef())
    5467           1 :     return Chain;
    5468             : 
    5469             :   // Expand memset to a series of load/store ops if the size operand
    5470             :   // falls below a certain threshold.
    5471       22907 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5472             :   std::vector<EVT> MemOps;
    5473             :   bool DstAlignCanChange = false;
    5474       22907 :   MachineFunction &MF = DAG.getMachineFunction();
    5475       22907 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    5476       22907 :   bool OptSize = shouldLowerMemFuncForSize(MF);
    5477             :   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
    5478         197 :   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
    5479             :     DstAlignCanChange = true;
    5480             :   bool IsZeroVal =
    5481       45716 :     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
    5482       45814 :   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
    5483             :                                 Size, (DstAlignCanChange ? 0 : Align), 0,
    5484             :                                 true, IsZeroVal, false, true,
    5485             :                                 DstPtrInfo.getAddrSpace(), ~0u,
    5486             :                                 DAG, TLI))
    5487         964 :     return SDValue();
    5488             : 
    5489       21943 :   if (DstAlignCanChange) {
    5490         132 :     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
    5491         264 :     unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
    5492         132 :     if (NewAlign > Align) {
    5493             :       // Give the stack frame object a larger alignment if needed.
    5494         232 :       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
    5495             :         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
    5496             :       Align = NewAlign;
    5497             :     }
    5498             :   }
    5499             : 
    5500             :   SmallVector<SDValue, 8> OutChains;
    5501             :   uint64_t DstOff = 0;
    5502       43886 :   unsigned NumMemOps = MemOps.size();
    5503             : 
    5504             :   // Find the largest store and generate the bit pattern for it.
    5505       21943 :   EVT LargestVT = MemOps[0];
    5506       65563 :   for (unsigned i = 1; i < NumMemOps; i++)
    5507       43620 :     if (MemOps[i].bitsGT(LargestVT))
    5508           0 :       LargestVT = MemOps[i];
    5509       21943 :   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
    5510             : 
    5511      109449 :   for (unsigned i = 0; i < NumMemOps; i++) {
    5512       87506 :     EVT VT = MemOps[i];
    5513       43753 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5514       43753 :     if (VTSize > Size) {
    5515             :       // Issuing an unaligned load / store pair  that overlaps with the previous
    5516             :       // pair. Adjust the offset accordingly.
    5517             :       assert(i == NumMemOps-1 && i != 0);
    5518           4 :       DstOff -= VTSize - Size;
    5519             :     }
    5520             : 
    5521             :     // If this store is smaller than the largest store see whether we can get
    5522             :     // the smaller value for free with a truncate.
    5523       43753 :     SDValue Value = MemSetValue;
    5524       43753 :     if (VT.bitsLT(LargestVT)) {
    5525        1507 :       if (!LargestVT.isVector() && !VT.isVector() &&
    5526          28 :           TLI.isTruncateFree(LargestVT, VT))
    5527          11 :         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
    5528             :       else
    5529        1440 :         Value = getMemsetValue(Src, VT, DAG, dl);
    5530             :     }
    5531             :     assert(Value.getValueType() == VT && "Value with wrong type.");
    5532             :     SDValue Store = DAG.getStore(
    5533             :         Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5534             :         DstPtrInfo.getWithOffset(DstOff), Align,
    5535       43753 :         isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone);
    5536       43753 :     OutChains.push_back(Store);
    5537       43753 :     DstOff += VT.getSizeInBits() / 8;
    5538       43753 :     Size -= VTSize;
    5539             :   }
    5540             : 
    5541       21943 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    5542             : }
    5543             : 
    5544        2019 : static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
    5545             :                                             unsigned AS) {
    5546             :   // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
    5547             :   // pointer operands can be losslessly bitcasted to pointers of address space 0
    5548        2019 :   if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) {
    5549           0 :     report_fatal_error("cannot lower memory intrinsic in address space " +
    5550             :                        Twine(AS));
    5551             :   }
    5552        2019 : }
    5553             : 
    5554        1750 : SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
    5555             :                                 SDValue Src, SDValue Size, unsigned Align,
    5556             :                                 bool isVol, bool AlwaysInline, bool isTailCall,
    5557             :                                 MachinePointerInfo DstPtrInfo,
    5558             :                                 MachinePointerInfo SrcPtrInfo) {
    5559             :   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
    5560             : 
    5561             :   // Check to see if we should lower the memcpy to loads and stores first.
    5562             :   // For cases within the target-specified limits, this is the best choice.
    5563             :   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
    5564             :   if (ConstantSize) {
    5565             :     // Memcpy with size zero? Just return the original chain.
    5566        2966 :     if (ConstantSize->isNullValue())
    5567        1212 :       return Chain;
    5568             : 
    5569             :     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
    5570             :                                              ConstantSize->getZExtValue(),Align,
    5571        2956 :                                 isVol, false, DstPtrInfo, SrcPtrInfo);
    5572        1478 :     if (Result.getNode())
    5573        1202 :       return Result;
    5574             :   }
    5575             : 
    5576             :   // Then check to see if we should lower the memcpy with target-specific
    5577             :   // code. If the target chooses to do this, this is the next best.
    5578         543 :   if (TSI) {
    5579             :     SDValue Result = TSI->EmitTargetCodeForMemcpy(
    5580             :         *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
    5581         543 :         DstPtrInfo, SrcPtrInfo);
    5582         543 :     if (Result.getNode())
    5583         147 :       return Result;
    5584             :   }
    5585             : 
    5586             :   // If we really need inline code and the target declined to provide it,
    5587             :   // use a (potentially long) sequence of loads and stores.
    5588         396 :   if (AlwaysInline) {
    5589             :     assert(ConstantSize && "AlwaysInline requires a constant size!");
    5590             :     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
    5591             :                                    ConstantSize->getZExtValue(), Align, isVol,
    5592           6 :                                    true, DstPtrInfo, SrcPtrInfo);
    5593             :   }
    5594             : 
    5595         393 :   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
    5596         393 :   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
    5597             : 
    5598             :   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
    5599             :   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
    5600             :   // respect volatile, so they may do things like read or write memory
    5601             :   // beyond the given memory regions. But fixing this isn't easy, and most
    5602             :   // people don't care.
    5603             : 
    5604             :   // Emit a library call.
    5605             :   TargetLowering::ArgListTy Args;
    5606             :   TargetLowering::ArgListEntry Entry;
    5607         786 :   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
    5608         393 :   Entry.Node = Dst; Args.push_back(Entry);
    5609         393 :   Entry.Node = Src; Args.push_back(Entry);
    5610         393 :   Entry.Node = Size; Args.push_back(Entry);
    5611             :   // FIXME: pass in SDLoc
    5612         786 :   TargetLowering::CallLoweringInfo CLI(*this);
    5613             :   CLI.setDebugLoc(dl)
    5614             :       .setChain(Chain)
    5615         393 :       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
    5616        1179 :                     Dst.getValueType().getTypeForEVT(*getContext()),
    5617         393 :                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
    5618             :                                       TLI->getPointerTy(getDataLayout())),
    5619        1965 :                     std::move(Args))
    5620             :       .setDiscardResult()
    5621             :       .setTailCall(isTailCall);
    5622             : 
    5623         393 :   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
    5624         393 :   return CallResult.second;
    5625             : }
    5626             : 
    5627         239 : SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
    5628             :                                  SDValue Src, SDValue Size, unsigned Align,
    5629             :                                  bool isVol, bool isTailCall,
    5630             :                                  MachinePointerInfo DstPtrInfo,
    5631             :                                  MachinePointerInfo SrcPtrInfo) {
    5632             :   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
    5633             : 
    5634             :   // Check to see if we should lower the memmove to loads and stores first.
    5635             :   // For cases within the target-specified limits, this is the best choice.
    5636             :   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
    5637             :   if (ConstantSize) {
    5638             :     // Memmove with size zero? Just return the original chain.
    5639         166 :     if (ConstantSize->isNullValue())
    5640          26 :       return Chain;
    5641             : 
    5642             :     SDValue Result =
    5643             :       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
    5644             :                                ConstantSize->getZExtValue(), Align, isVol,
    5645         166 :                                false, DstPtrInfo, SrcPtrInfo);
    5646          83 :     if (Result.getNode())
    5647          26 :       return Result;
    5648             :   }
    5649             : 
    5650             :   // Then check to see if we should lower the memmove with target-specific
    5651             :   // code. If the target chooses to do this, this is the next best.
    5652         213 :   if (TSI) {
    5653             :     SDValue Result = TSI->EmitTargetCodeForMemmove(
    5654         213 :         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
    5655         213 :     if (Result.getNode())
    5656          48 :       return Result;
    5657             :   }
    5658             : 
    5659         165 :   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
    5660         165 :   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
    5661             : 
    5662             :   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
    5663             :   // not be safe.  See memcpy above for more details.
    5664             : 
    5665             :   // Emit a library call.
    5666             :   TargetLowering::ArgListTy Args;
    5667             :   TargetLowering::ArgListEntry Entry;
    5668         330 :   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
    5669         165 :   Entry.Node = Dst; Args.push_back(Entry);
    5670         165 :   Entry.Node = Src; Args.push_back(Entry);
    5671         165 :   Entry.Node = Size; Args.push_back(Entry);
    5672             :   // FIXME:  pass in SDLoc
    5673         330 :   TargetLowering::CallLoweringInfo CLI(*this);
    5674             :   CLI.setDebugLoc(dl)
    5675             :       .setChain(Chain)
    5676         165 :       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
    5677         495 :                     Dst.getValueType().getTypeForEVT(*getContext()),
    5678         165 :                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
    5679             :                                       TLI->getPointerTy(getDataLayout())),
    5680         825 :                     std::move(Args))
    5681             :       .setDiscardResult()
    5682             :       .setTailCall(isTailCall);
    5683             : 
    5684         165 :   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
    5685         165 :   return CallResult.second;
    5686             : }
    5687             : 
    5688       23028 : SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
    5689             :                                 SDValue Src, SDValue Size, unsigned Align,
    5690             :                                 bool isVol, bool isTailCall,
    5691             :                                 MachinePointerInfo DstPtrInfo) {
    5692             :   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
    5693             : 
    5694             :   // Check to see if we should lower the memset to stores first.
    5695             :   // For cases within the target-specified limits, this is the best choice.
    5696             :   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
    5697             :   if (ConstantSize) {
    5698             :     // Memset with size zero? Just return the original chain.
    5699       45832 :     if (ConstantSize->isNullValue())
    5700       21960 :       return Chain;
    5701             : 
    5702             :     SDValue Result =
    5703             :       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
    5704       45816 :                       Align, isVol, DstPtrInfo);
    5705             : 
    5706       22908 :     if (Result.getNode())
    5707       21944 :       return Result;
    5708             :   }
    5709             : 
    5710             :   // Then check to see if we should lower the memset with target-specific
    5711             :   // code. If the target chooses to do this, this is the next best.
    5712        1076 :   if (TSI) {
    5713             :     SDValue Result = TSI->EmitTargetCodeForMemset(
    5714        1076 :         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
    5715        1076 :     if (Result.getNode())
    5716         173 :       return Result;
    5717             :   }
    5718             : 
    5719         903 :   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
    5720             : 
    5721             :   // Emit a library call.
    5722        1806 :   Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
    5723             :   TargetLowering::ArgListTy Args;
    5724             :   TargetLowering::ArgListEntry Entry;
    5725         903 :   Entry.Node = Dst; Entry.Ty = IntPtrTy;
    5726         903 :   Args.push_back(Entry);
    5727         903 :   Entry.Node = Src;
    5728        1806 :   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
    5729         903 :   Args.push_back(Entry);
    5730         903 :   Entry.Node = Size;
    5731         903 :   Entry.Ty = IntPtrTy;
    5732         903 :   Args.push_back(Entry);
    5733             : 
    5734             :   // FIXME: pass in SDLoc
    5735        1806 :   TargetLowering::CallLoweringInfo CLI(*this);
    5736             :   CLI.setDebugLoc(dl)
    5737             :       .setChain(Chain)
    5738         903 :       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
    5739        2709 :                     Dst.getValueType().getTypeForEVT(*getContext()),
    5740         903 :                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
    5741             :                                       TLI->getPointerTy(getDataLayout())),
    5742        4515 :                     std::move(Args))
    5743             :       .setDiscardResult()
    5744             :       .setTailCall(isTailCall);
    5745             : 
    5746         903 :   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
    5747         903 :   return CallResult.second;
    5748             : }
    5749             : 
    5750       10539 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5751             :                                 SDVTList VTList, ArrayRef<SDValue> Ops,
    5752             :                                 MachineMemOperand *MMO) {
    5753             :   FoldingSetNodeID ID;
    5754       10539 :   ID.AddInteger(MemVT.getRawBits());
    5755       10539 :   AddNodeIDNode(ID, Opcode, VTList, Ops);
    5756       10539 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    5757       10539 :   void* IP = nullptr;
    5758       10539 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    5759           0 :     cast<AtomicSDNode>(E)->refineAlignment(MMO);
    5760           0 :     return SDValue(E, 0);
    5761             :   }
    5762             : 
    5763       10539 :   auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
    5764       10539 :                                     VTList, MemVT, MMO);
    5765       10539 :   createOperands(N, Ops);
    5766             : 
    5767       10539 :   CSEMap.InsertNode(N, IP);
    5768       10539 :   InsertNode(N);
    5769       10539 :   return SDValue(N, 0);
    5770             : }
    5771             : 
    5772        1162 : SDValue SelectionDAG::getAtomicCmpSwap(
    5773             :     unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain,
    5774             :     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
    5775             :     unsigned Alignment, AtomicOrdering SuccessOrdering,
    5776             :     AtomicOrdering FailureOrdering, SyncScope::ID SSID) {
    5777             :   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
    5778             :          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
    5779             :   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
    5780             : 
    5781        1162 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5782        1162 :     Alignment = getEVTAlignment(MemVT);
    5783             : 
    5784        1162 :   MachineFunction &MF = getMachineFunction();
    5785             : 
    5786             :   // FIXME: Volatile isn't really correct; we should keep track of atomic
    5787             :   // orderings in the memoperand.
    5788             :   auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad |
    5789             :                MachineMemOperand::MOStore;
    5790             :   MachineMemOperand *MMO =
    5791        2324 :     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
    5792        1162 :                             AAMDNodes(), nullptr, SSID, SuccessOrdering,
    5793        1162 :                             FailureOrdering);
    5794             : 
    5795        1162 :   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO);
    5796             : }
    5797             : 
    5798        2898 : SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
    5799             :                                        EVT MemVT, SDVTList VTs, SDValue Chain,
    5800             :                                        SDValue Ptr, SDValue Cmp, SDValue Swp,
    5801             :                                        MachineMemOperand *MMO) {
    5802             :   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
    5803             :          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
    5804             :   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
    5805             : 
    5806        2898 :   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
    5807        2898 :   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
    5808             : }
    5809             : 
    5810        5234 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5811             :                                 SDValue Chain, SDValue Ptr, SDValue Val,
    5812             :                                 const Value *PtrVal, unsigned Alignment,
    5813             :                                 AtomicOrdering Ordering,
    5814             :                                 SyncScope::ID SSID) {
    5815        5234 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5816        4633 :     Alignment = getEVTAlignment(MemVT);
    5817             : 
    5818        5234 :   MachineFunction &MF = getMachineFunction();
    5819             :   // An atomic store does not load. An atomic load does not store.
    5820             :   // (An atomicrmw obviously both loads and stores.)
    5821             :   // For now, atomics are considered to be volatile always, and they are
    5822             :   // chained as such.
    5823             :   // FIXME: Volatile isn't really correct; we should keep track of atomic
    5824             :   // orderings in the memoperand.
    5825             :   auto Flags = MachineMemOperand::MOVolatile;
    5826        5234 :   if (Opcode != ISD::ATOMIC_STORE)
    5827             :     Flags |= MachineMemOperand::MOLoad;
    5828        5234 :   if (Opcode != ISD::ATOMIC_LOAD)
    5829             :     Flags |= MachineMemOperand::MOStore;
    5830             : 
    5831             :   MachineMemOperand *MMO =
    5832       15702 :     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
    5833        5234 :                             MemVT.getStoreSize(), Alignment, AAMDNodes(),
    5834        5234 :                             nullptr, SSID, Ordering);
    5835             : 
    5836        5234 :   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
    5837             : }
    5838             : 
    5839        6981 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5840             :                                 SDValue Chain, SDValue Ptr, SDValue Val,
    5841             :                                 MachineMemOperand *MMO) {
    5842             :   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
    5843             :           Opcode == ISD::ATOMIC_LOAD_SUB ||
    5844             :           Opcode == ISD::ATOMIC_LOAD_AND ||
    5845             :           Opcode == ISD::ATOMIC_LOAD_CLR ||
    5846             :           Opcode == ISD::ATOMIC_LOAD_OR ||
    5847             :           Opcode == ISD::ATOMIC_LOAD_XOR ||
    5848             :           Opcode == ISD::ATOMIC_LOAD_NAND ||
    5849             :           Opcode == ISD::ATOMIC_LOAD_MIN ||
    5850             :           Opcode == ISD::ATOMIC_LOAD_MAX ||
    5851             :           Opcode == ISD::ATOMIC_LOAD_UMIN ||
    5852             :           Opcode == ISD::ATOMIC_LOAD_UMAX ||
    5853             :           Opcode == ISD::ATOMIC_SWAP ||
    5854             :           Opcode == ISD::ATOMIC_STORE) &&
    5855             :          "Invalid Atomic Op");
    5856             : 
    5857       13962 :   EVT VT = Val.getValueType();
    5858             : 
    5859        6981 :   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
    5860       13962 :                                                getVTList(VT, MVT::Other);
    5861        6981 :   SDValue Ops[] = {Chain, Ptr, Val};
    5862        6981 :   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
    5863             : }
    5864             : 
    5865         660 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5866             :                                 EVT VT, SDValue Chain, SDValue Ptr,
    5867             :                                 MachineMemOperand *MMO) {
    5868             :   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
    5869             : 
    5870         660 :   SDVTList VTs = getVTList(VT, MVT::Other);
    5871         660 :   SDValue Ops[] = {Chain, Ptr};
    5872         660 :   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
    5873             : }
    5874             : 
    5875             : /// getMergeValues - Create a MERGE_VALUES node from the given operands.
    5876      326834 : SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
    5877      326834 :   if (Ops.size() == 1)
    5878      275003 :     return Ops[0];
    5879             : 
    5880             :   SmallVector<EVT, 4> VTs;
    5881       51831 :   VTs.reserve(Ops.size());
    5882      263335 :   for (unsigned i = 0; i < Ops.size(); ++i)
    5883      211504 :     VTs.push_back(Ops[i].getValueType());
    5884       51831 :   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
    5885             : }
    5886             : 
    5887        8390 : SDValue SelectionDAG::getMemIntrinsicNode(
    5888             :     unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
    5889             :     EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align,
    5890             :     MachineMemOperand::Flags Flags, unsigned Size) {
    5891        8390 :   if (Align == 0)  // Ensure that codegen never sees alignment 0
    5892        4283 :     Align = getEVTAlignment(MemVT);
    5893             : 
    5894        8390 :   if (!Size)
    5895             :     Size = MemVT.getStoreSize();
    5896             : 
    5897        8390 :   MachineFunction &MF = getMachineFunction();
    5898             :   MachineMemOperand *MMO =
    5899        8390 :     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
    5900             : 
    5901        8390 :   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
    5902             : }
    5903             : 
    5904       15132 : SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
    5905             :                                           SDVTList VTList,
    5906             :                                           ArrayRef<SDValue> Ops, EVT MemVT,
    5907             :                                           MachineMemOperand *MMO) {
    5908             :   assert((Opcode == ISD::INTRINSIC_VOID ||
    5909             :           Opcode == ISD::INTRINSIC_W_CHAIN ||
    5910             :           Opcode == ISD::PREFETCH ||
    5911             :           Opcode == ISD::LIFETIME_START ||
    5912             :           Opcode == ISD::LIFETIME_END ||
    5913             :           ((int)Opcode <= std::numeric_limits<int>::max() &&
    5914             :            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
    5915             :          "Opcode is not a memory-accessing opcode!");
    5916             : 
    5917             :   // Memoize the node unless it returns a flag.
    5918             :   MemIntrinsicSDNode *N;
    5919       15132 :   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
    5920             :     FoldingSetNodeID ID;
    5921       14117 :     AddNodeIDNode(ID, Opcode, VTList, Ops);
    5922       28234 :     ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
    5923             :         Opcode, dl.getIROrder(), VTList, MemVT, MMO));
    5924       14117 :     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    5925       14117 :     void *IP = nullptr;
    5926       14117 :     if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    5927           0 :       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
    5928           0 :       return SDValue(E, 0);
    5929             :     }
    5930             : 
    5931       28234 :     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
    5932             :                                       VTList, MemVT, MMO);
    5933       14117 :     createOperands(N, Ops);
    5934             : 
    5935       14117 :   CSEMap.InsertNode(N, IP);
    5936             :   } else {
    5937        2030 :     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
    5938             :                                       VTList, MemVT, MMO);
    5939        1015 :     createOperands(N, Ops);
    5940             :   }
    5941       15132 :   InsertNode(N);
    5942       15132 :   return SDValue(N, 0);
    5943             : }
    5944             : 
    5945             : /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
    5946             : /// MachinePointerInfo record from it.  This is particularly useful because the
    5947             : /// code generator has many cases where it doesn't bother passing in a
    5948             : /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
    5949       28363 : static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
    5950             :                                            SelectionDAG &DAG, SDValue Ptr,
    5951             :                                            int64_t Offset = 0) {
    5952             :   // If this is FI+Offset, we can model it.
    5953             :   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
    5954             :     return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
    5955       10799 :                                              FI->getIndex(), Offset);
    5956             : 
    5957             :   // If this is (FI+Offset1)+Offset2, we can model it.
    5958             :   if (Ptr.getOpcode() != ISD::ADD ||
    5959       17564 :       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
    5960             :       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
    5961       15347 :     return Info;
    5962             : 
    5963        2217 :   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
    5964             :   return MachinePointerInfo::getFixedStack(
    5965             :       DAG.getMachineFunction(), FI,
    5966        4434 :       Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
    5967             : }
    5968             : 
    5969             : /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
    5970             : /// MachinePointerInfo record from it.  This is particularly useful because the
    5971             : /// code generator has many cases where it doesn't bother passing in a
    5972             : /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
    5973       12224 : static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
    5974             :                                            SelectionDAG &DAG, SDValue Ptr,
    5975             :                                            SDValue OffsetOp) {
    5976             :   // If the 'Offset' value isn't a constant, we can't handle this.
    5977             :   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
    5978          18 :     return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
    5979       12215 :   if (OffsetOp.isUndef())
    5980       12215 :     return InferPointerInfo(Info, DAG, Ptr);
    5981           0 :   return Info;
    5982             : }
    5983             : 
    5984      611142 : SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
    5985             :                               EVT VT, const SDLoc &dl, SDValue Chain,
    5986             :                               SDValue Ptr, SDValue Offset,
    5987             :                               MachinePointerInfo PtrInfo, EVT MemVT,
    5988             :                               unsigned Alignment,
    5989             :                               MachineMemOperand::Flags MMOFlags,
    5990             :                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
    5991             :   assert(Chain.getValueType() == MVT::Other &&
    5992             :         "Invalid chain type");
    5993      611142 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5994      101125 :     Alignment = getEVTAlignment(MemVT);
    5995             : 
    5996             :   MMOFlags |= MachineMemOperand::MOLoad;
    5997             :   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
    5998             :   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
    5999             :   // clients.
    6000      611142 :   if (PtrInfo.V.isNull())
    6001       12224 :     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
    6002             : 
    6003      611142 :   MachineFunction &MF = getMachineFunction();
    6004      611142 :   MachineMemOperand *MMO = MF.getMachineMemOperand(
    6005      611142 :       PtrInfo, MMOFlags, MemVT.getStoreSize(), Alignment, AAInfo, Ranges);
    6006      611142 :   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
    6007             : }
    6008             : 
    6009      830189 : SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
    6010             :                               EVT VT, const SDLoc &dl, SDValue Chain,
    6011             :                               SDValue Ptr, SDValue Offset, EVT MemVT,
    6012             :                               MachineMemOperand *MMO) {
    6013      833044 :   if (VT == MemVT) {
    6014             :     ExtType = ISD::NON_EXTLOAD;
    6015             :   } else if (ExtType == ISD::NON_EXTLOAD) {
    6016             :     assert(VT == MemVT && "Non-extending load from different memory type!");
    6017             :   } else {
    6018             :     // Extending load.
    6019             :     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
    6020             :            "Should only be an extending load, not truncating!");
    6021             :     assert(VT.isInteger() == MemVT.isInteger() &&
    6022             :            "Cannot convert from FP to Int or Int -> FP!");
    6023             :     assert(VT.isVector() == MemVT.isVector() &&
    6024             :            "Cannot use an ext load to convert to or from a vector!");
    6025             :     assert((!VT.isVector() ||
    6026             :             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
    6027             :            "Cannot use an ext load to change the number of vector elements!");
    6028             :   }
    6029             : 
    6030             :   bool Indexed = AM != ISD::UNINDEXED;
    6031             :   assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
    6032             : 
    6033             :   SDVTList VTs = Indexed ?
    6034     1660794 :     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
    6035      830189 :   SDValue Ops[] = { Chain, Ptr, Offset };
    6036             :   FoldingSetNodeID ID;
    6037      830189 :   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
    6038      830189 :   ID.AddInteger(MemVT.getRawBits());
    6039     1660378 :   ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
    6040             :       dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
    6041      830189 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6042      830189 :   void *IP = nullptr;
    6043      830189 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6044       16846 :     cast<LoadSDNode>(E)->refineAlignment(MMO);
    6045       16846 :     return SDValue(E, 0);
    6046             :   }
    6047      813343 :   auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
    6048      813343 :                                   ExtType, MemVT, MMO);
    6049      813343 :   createOperands(N, Ops);
    6050             : 
    6051      813343 :   CSEMap.InsertNode(N, IP);
    6052      813343 :   InsertNode(N);
    6053             :   SDValue V(N, 0);
    6054             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6055      813343 :   return V;
    6056             : }
    6057             : 
    6058      554732 : SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
    6059             :                               SDValue Ptr, MachinePointerInfo PtrInfo,
    6060             :                               unsigned Alignment,
    6061             :                               MachineMemOperand::Flags MMOFlags,
    6062             :                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
    6063     1109464 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    6064             :   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
    6065      554732 :                  PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
    6066             : }
    6067             : 
    6068      192854 : SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
    6069             :                               SDValue Ptr, MachineMemOperand *MMO) {
    6070      385708 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    6071             :   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
    6072      192854 :                  VT, MMO);
    6073             : }
    6074             : 
    6075       38504 : SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
    6076             :                                  EVT VT, SDValue Chain, SDValue Ptr,
    6077             :                                  MachinePointerInfo PtrInfo, EVT MemVT,
    6078             :                                  unsigned Alignment,
    6079             :                                  MachineMemOperand::Flags MMOFlags,
    6080             :                                  const AAMDNodes &AAInfo) {
    6081       77008 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    6082             :   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
    6083       38504 :                  MemVT, Alignment, MMOFlags, AAInfo);
    6084             : }
    6085             : 
    6086       24930 : SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
    6087             :                                  EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
    6088             :                                  MachineMemOperand *MMO) {
    6089       49860 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    6090             :   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
    6091       24930 :                  MemVT, MMO);
    6092             : }
    6093             : 
    6094         416 : SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
    6095             :                                      SDValue Base, SDValue Offset,
    6096             :                                      ISD::MemIndexedMode AM) {
    6097             :   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
    6098             :   assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
    6099             :   // Don't propagate the invariant or dereferenceable flags.
    6100             :   auto MMOFlags =
    6101         416 :       LD->getMemOperand()->getFlags() &
    6102             :       ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
    6103             :   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
    6104         416 :                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
    6105             :                  LD->getMemoryVT(), LD->getAlignment(), MMOFlags,
    6106        1248 :                  LD->getAAInfo());
    6107             : }
    6108             : 
    6109      712391 : SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    6110             :                                SDValue Ptr, MachinePointerInfo PtrInfo,
    6111             :                                unsigned Alignment,
    6112             :                                MachineMemOperand::Flags MMOFlags,
    6113             :                                const AAMDNodes &AAInfo) {
    6114             :   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
    6115      712391 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    6116      189651 :     Alignment = getEVTAlignment(Val.getValueType());
    6117             : 
    6118             :   MMOFlags |= MachineMemOperand::MOStore;
    6119             :   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
    6120             : 
    6121      712391 :   if (PtrInfo.V.isNull())
    6122       14337 :     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
    6123             : 
    6124      712391 :   MachineFunction &MF = getMachineFunction();
    6125      712391 :   MachineMemOperand *MMO = MF.getMachineMemOperand(
    6126     1424782 :       PtrInfo, MMOFlags, Val.getValueType().getStoreSize(), Alignment, AAInfo);
    6127      712391 :   return getStore(Chain, dl, Val, Ptr, MMO);
    6128             : }
    6129             : 
    6130     1048823 : SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    6131             :                                SDValue Ptr, MachineMemOperand *MMO) {
    6132             :   assert(Chain.getValueType() == MVT::Other &&
    6133             :         "Invalid chain type");
    6134     2097646 :   EVT VT = Val.getValueType();
    6135     1048823 :   SDVTList VTs = getVTList(MVT::Other);
    6136     2097646 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    6137     1048823 :   SDValue Ops[] = { Chain, Val, Ptr, Undef };
    6138             :   FoldingSetNodeID ID;
    6139     1048823 :   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
    6140     1048823 :   ID.AddInteger(VT.getRawBits());
    6141     2097646 :   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
    6142             :       dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
    6143     1048823 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6144     1048823 :   void *IP = nullptr;
    6145     1048823 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6146        5371 :     cast<StoreSDNode>(E)->refineAlignment(MMO);
    6147        5371 :     return SDValue(E, 0);
    6148             :   }
    6149     1043452 :   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    6150     1043452 :                                    ISD::UNINDEXED, false, VT, MMO);
    6151     1043452 :   createOperands(N, Ops);
    6152             : 
    6153     1043452 :   CSEMap.InsertNode(N, IP);
    6154     1043452 :   InsertNode(N);
    6155             :   SDValue V(N, 0);
    6156             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6157     1043452 :   return V;
    6158             : }
    6159             : 
    6160       36613 : SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    6161             :                                     SDValue Ptr, MachinePointerInfo PtrInfo,
    6162             :                                     EVT SVT, unsigned Alignment,
    6163             :                                     MachineMemOperand::Flags MMOFlags,
    6164             :                                     const AAMDNodes &AAInfo) {
    6165             :   assert(Chain.getValueType() == MVT::Other &&
    6166             :         "Invalid chain type");
    6167       36613 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    6168         577 :     Alignment = getEVTAlignment(SVT);
    6169             : 
    6170             :   MMOFlags |= MachineMemOperand::MOStore;
    6171             :   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
    6172             : 
    6173       36613 :   if (PtrInfo.V.isNull())
    6174        1802 :     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
    6175             : 
    6176       36613 :   MachineFunction &MF = getMachineFunction();
    6177       36613 :   MachineMemOperand *MMO = MF.getMachineMemOperand(
    6178       36613 :       PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
    6179       36613 :   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
    6180             : }
    6181             : 
    6182       48548 : SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    6183             :                                     SDValue Ptr, EVT SVT,
    6184             :                                     MachineMemOperand *MMO) {
    6185       48548 :   EVT VT = Val.getValueType();
    6186             : 
    6187             :   assert(Chain.getValueType() == MVT::Other &&
    6188             :         "Invalid chain type");
    6189       49078 :   if (VT == SVT)
    6190       28026 :     return getStore(Chain, dl, Val, Ptr, MMO);
    6191             : 
    6192             :   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
    6193             :          "Should only be a truncating store, not extending!");
    6194             :   assert(VT.isInteger() == SVT.isInteger() &&
    6195             :          "Can't do FP-INT conversion!");
    6196             :   assert(VT.isVector() == SVT.isVector() &&
    6197             :          "Cannot use trunc store to convert to or from a vector!");
    6198             :   assert((!VT.isVector() ||
    6199             :           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
    6200             :          "Cannot use trunc store to change the number of vector elements!");
    6201             : 
    6202       20522 :   SDVTList VTs = getVTList(MVT::Other);
    6203       41044 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    6204       20522 :   SDValue Ops[] = { Chain, Val, Ptr, Undef };
    6205             :   FoldingSetNodeID ID;
    6206       20522 :   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
    6207       20522 :   ID.AddInteger(SVT.getRawBits());
    6208       41044 :   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
    6209             :       dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
    6210       20522 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6211       20522 :   void *IP = nullptr;
    6212       20522 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6213         639 :     cast<StoreSDNode>(E)->refineAlignment(MMO);
    6214         639 :     return SDValue(E, 0);
    6215             :   }
    6216       19883 :   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    6217       19883 :                                    ISD::UNINDEXED, true, SVT, MMO);
    6218       19883 :   createOperands(N, Ops);
    6219             : 
    6220       19883 :   CSEMap.InsertNode(N, IP);
    6221       19883 :   InsertNode(N);
    6222             :   SDValue V(N, 0);
    6223             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6224       19883 :   return V;
    6225             : }
    6226             : 
    6227         236 : SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
    6228             :                                       SDValue Base, SDValue Offset,
    6229             :                                       ISD::MemIndexedMode AM) {
    6230             :   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
    6231             :   assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
    6232         472 :   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
    6233         236 :   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
    6234             :   FoldingSetNodeID ID;
    6235         236 :   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
    6236         236 :   ID.AddInteger(ST->getMemoryVT().getRawBits());
    6237         236 :   ID.AddInteger(ST->getRawSubclassData());
    6238         472 :   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
    6239         236 :   void *IP = nullptr;
    6240         236 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
    6241           0 :     return SDValue(E, 0);
    6242             : 
    6243         236 :   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
    6244         472 :                                    ST->isTruncatingStore(), ST->getMemoryVT(),
    6245         472 :                                    ST->getMemOperand());
    6246         236 :   createOperands(N, Ops);
    6247             : 
    6248         236 :   CSEMap.InsertNode(N, IP);
    6249         236 :   InsertNode(N);
    6250             :   SDValue V(N, 0);
    6251             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6252         236 :   return V;
    6253             : }
    6254             : 
    6255         352 : SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
    6256             :                                     SDValue Ptr, SDValue Mask, SDValue Src0,
    6257             :                                     EVT MemVT, MachineMemOperand *MMO,
    6258             :                                     ISD::LoadExtType ExtTy, bool isExpanding) {
    6259         352 :   SDVTList VTs = getVTList(VT, MVT::Other);
    6260         352 :   SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
    6261             :   FoldingSetNodeID ID;
    6262         352 :   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
    6263         352 :   ID.AddInteger(VT.getRawBits());
    6264         704 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
    6265             :       dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO));
    6266         352 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6267         352 :   void *IP = nullptr;
    6268         352 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6269           0 :     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
    6270           0 :     return SDValue(E, 0);
    6271             :   }
    6272         352 :   auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    6273         352 :                                         ExtTy, isExpanding, MemVT, MMO);
    6274         352 :   createOperands(N, Ops);
    6275             : 
    6276         352 :   CSEMap.InsertNode(N, IP);
    6277         352 :   InsertNode(N);
    6278             :   SDValue V(N, 0);
    6279             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6280         352 :   return V;
    6281             : }
    6282             : 
    6283         201 : SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
    6284             :                                      SDValue Val, SDValue Ptr, SDValue Mask,
    6285             :                                      EVT MemVT, MachineMemOperand *MMO,
    6286             :                                      bool IsTruncating, bool IsCompressing) {
    6287             :   assert(Chain.getValueType() == MVT::Other &&
    6288             :         "Invalid chain type");
    6289         201 :   EVT VT = Val.getValueType();
    6290         201 :   SDVTList VTs = getVTList(MVT::Other);
    6291         201 :   SDValue Ops[] = { Chain, Ptr, Mask, Val };
    6292             :   FoldingSetNodeID ID;
    6293         201 :   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
    6294         201 :   ID.AddInteger(VT.getRawBits());
    6295         402 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
    6296             :       dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO));
    6297         201 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6298         201 :   void *IP = nullptr;
    6299         201 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6300           0 :     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
    6301           0 :     return SDValue(E, 0);
    6302             :   }
    6303         201 :   auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    6304         201 :                                          IsTruncating, IsCompressing, MemVT, MMO);
    6305         201 :   createOperands(N, Ops);
    6306             : 
    6307         201 :   CSEMap.InsertNode(N, IP);
    6308         201 :   InsertNode(N);
    6309             :   SDValue V(N, 0);
    6310             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6311         201 :   return V;
    6312             : }
    6313             : 
    6314         430 : SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
    6315             :                                       ArrayRef<SDValue> Ops,
    6316             :                                       MachineMemOperand *MMO) {
    6317             :   assert(Ops.size() == 6 && "Incompatible number of operands");
    6318             : 
    6319             :   FoldingSetNodeID ID;
    6320         430 :   AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
    6321         430 :   ID.AddInteger(VT.getRawBits());
    6322         860 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
    6323             :       dl.getIROrder(), VTs, VT, MMO));
    6324         430 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6325         430 :   void *IP = nullptr;
    6326         430 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6327           0 :     cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
    6328           0 :     return SDValue(E, 0);
    6329             :   }
    6330             : 
    6331         430 :   auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
    6332         430 :                                           VTs, VT, MMO);
    6333         430 :   createOperands(N, Ops);
    6334             : 
    6335             :   assert(N->getValue().getValueType() == N->getValueType(0) &&
    6336             :          "Incompatible type of the PassThru value in MaskedGatherSDNode");
    6337             :   assert(N->getMask().getValueType().getVectorNumElements() ==
    6338             :              N->getValueType(0).getVectorNumElements() &&
    6339             :          "Vector width mismatch between mask and data");
    6340             :   assert(N->getIndex().getValueType().getVectorNumElements() ==
    6341             :              N->getValueType(0).getVectorNumElements() &&
    6342             :          "Vector width mismatch between index and data");
    6343             :   assert(isa<ConstantSDNode>(N->getScale()) &&
    6344             :          cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
    6345             :          "Scale should be a constant power of 2");
    6346             : 
    6347         430 :   CSEMap.InsertNode(N, IP);
    6348         430 :   InsertNode(N);
    6349             :   SDValue V(N, 0);
    6350             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6351         430 :   return V;
    6352             : }
    6353             : 
    6354         148 : SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
    6355             :                                        ArrayRef<SDValue> Ops,
    6356             :                                        MachineMemOperand *MMO) {
    6357             :   assert(Ops.size() == 6 && "Incompatible number of operands");
    6358             : 
    6359             :   FoldingSetNodeID ID;
    6360         148 :   AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
    6361         148 :   ID.AddInteger(VT.getRawBits());
    6362         296 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
    6363             :       dl.getIROrder(), VTs, VT, MMO));
    6364         148 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6365         148 :   void *IP = nullptr;
    6366         148 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6367           0 :     cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
    6368           0 :     return SDValue(E, 0);
    6369             :   }
    6370         148 :   auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
    6371         148 :                                            VTs, VT, MMO);
    6372         148 :   createOperands(N, Ops);
    6373             : 
    6374             :   assert(N->getMask().getValueType().getVectorNumElements() ==
    6375             :              N->getValue().getValueType().getVectorNumElements() &&
    6376             :          "Vector width mismatch between mask and data");
    6377             :   assert(N->getIndex().getValueType().getVectorNumElements() ==
    6378             :              N->getValue().getValueType().getVectorNumElements() &&
    6379             :          "Vector width mismatch between index and data");
    6380             :   assert(isa<ConstantSDNode>(N->getScale()) &&
    6381             :          cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
    6382             :          "Scale should be a constant power of 2");
    6383             : 
    6384         148 :   CSEMap.InsertNode(N, IP);
    6385         148 :   InsertNode(N);
    6386             :   SDValue V(N, 0);
    6387             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6388         148 :   return V;
    6389             : }
    6390             : 
    6391         245 : SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
    6392             :                                SDValue Ptr, SDValue SV, unsigned Align) {
    6393         490 :   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
    6394         245 :   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
    6395             : }
    6396             : 
    6397        5279 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    6398             :                               ArrayRef<SDUse> Ops) {
    6399        5279 :   switch (Ops.size()) {
    6400           0 :   case 0: return getNode(Opcode, DL, VT);
    6401          36 :   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
    6402        2649 :   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
    6403         104 :   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
    6404             :   default: break;
    6405             :   }
    6406             : 
    6407             :   // Copy from an SDUse array into an SDValue array for use with
    6408             :   // the regular getNode logic.
    6409             :   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
    6410        2542 :   return getNode(Opcode, DL, VT, NewOps);
    6411             : }
    6412             : 
    6413     3310693 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    6414             :                               ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
    6415     3310693 :   unsigned NumOps = Ops.size();
    6416     3310693 :   switch (NumOps) {
    6417           0 :   case 0: return getNode(Opcode, DL, VT);
    6418     1255094 :   case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
    6419     1693992 :   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
    6420      639260 :   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
    6421             :   default: break;
    6422             :   }
    6423             : 
    6424      888973 :   switch (Opcode) {
    6425             :   default: break;
    6426        6490 :   case ISD::CONCAT_VECTORS:
    6427             :     // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
    6428        6490 :     if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
    6429         429 :       return V;
    6430        6061 :     break;
    6431             :   case ISD::SELECT_CC:
    6432             :     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
    6433             :     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
    6434             :            "LHS and RHS of condition must have same type!");
    6435             :     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
    6436             :            "True and False arms of SelectCC must have same type!");
    6437             :     assert(Ops[2].getValueType() == VT &&
    6438             :            "select_cc node must be of same type as true and false value!");
    6439             :     break;
    6440             :   case ISD::BR_CC:
    6441             :     assert(NumOps == 5 && "BR_CC takes 5 operands!");
    6442             :     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
    6443             :            "LHS/RHS of comparison should match types!");
    6444             :     break;
    6445             :   }
    6446             : 
    6447             :   // Memoize nodes.
    6448             :   SDNode *N;
    6449      888544 :   SDVTList VTs = getVTList(VT);
    6450             : 
    6451      888544 :   if (VT != MVT::Glue) {
    6452             :     FoldingSetNodeID ID;
    6453      888544 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    6454      888544 :     void *IP = nullptr;
    6455             : 
    6456      888544 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    6457      279547 :       return SDValue(E, 0);
    6458             : 
    6459     1217994 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    6460      608997 :     createOperands(N, Ops);
    6461             : 
    6462      608997 :     CSEMap.InsertNode(N, IP);
    6463             :   } else {
    6464           0 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    6465           0 :     createOperands(N, Ops);
    6466             :   }
    6467             : 
    6468      608997 :   InsertNode(N);
    6469             :   SDValue V(N, 0);
    6470             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6471      608997 :   return V;
    6472             : }
    6473             : 
    6474       13617 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
    6475             :                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
    6476       13617 :   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
    6477             : }
    6478             : 
    6479     3058397 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6480             :                               ArrayRef<SDValue> Ops) {
    6481     3058397 :   if (VTList.NumVTs == 1)
    6482      570885 :     return getNode(Opcode, DL, VTList.VTs[0], Ops);
    6483             : 
    6484             : #if 0
    6485             :   switch (Opcode) {
    6486             :   // FIXME: figure out how to safely handle things like
    6487             :   // int foo(int x) { return 1 << (x & 255); }
    6488             :   // int bar() { return foo(256); }
    6489             :   case ISD::SRA_PARTS:
    6490             :   case ISD::SRL_PARTS:
    6491             :   case ISD::SHL_PARTS:
    6492             :     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
    6493             :         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
    6494             :       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
    6495             :     else if (N3.getOpcode() == ISD::AND)
    6496             :       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
    6497             :         // If the and is only masking out bits that cannot effect the shift,
    6498             :         // eliminate the and.
    6499             :         unsigned NumBits = VT.getScalarSizeInBits()*2;
    6500             :         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
    6501             :           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
    6502             :       }
    6503             :     break;
    6504             :   }
    6505             : #endif
    6506             : 
    6507             :   // Memoize the node unless it returns a flag.
    6508             :   SDNode *N;
    6509     2487512 :   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
    6510             :     FoldingSetNodeID ID;
    6511     1196102 :     AddNodeIDNode(ID, Opcode, VTList, Ops);
    6512     1196102 :     void *IP = nullptr;
    6513     1196102 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    6514       58274 :       return SDValue(E, 0);
    6515             : 
    6516     2275656 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
    6517     1137828 :     createOperands(N, Ops);
    6518     1137828 :     CSEMap.InsertNode(N, IP);
    6519             :   } else {
    6520     2582820 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
    6521     1291410 :     createOperands(N, Ops);
    6522             :   }
    6523     2429238 :   InsertNode(N);
    6524             :   SDValue V(N, 0);
    6525             :   NewSDValueDbgMsg(V, "Creating new node: ", this);
    6526     2429238 :   return V;
    6527             : }
    6528             : 
    6529           0 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
    6530             :                               SDVTList VTList) {
    6531           0 :   return getNode(Opcode, DL, VTList, None);
    6532             : }
    6533             : 
    6534        3511 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6535             :                               SDValue N1) {
    6536        3511 :   SDValue Ops[] = { N1 };
    6537        3511 :   return getNode(Opcode, DL, VTList, Ops);
    6538             : }
    6539             : 
    6540      293920 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6541             :                               SDValue N1, SDValue N2) {
    6542      293920 :   SDValue Ops[] = { N1, N2 };
    6543      293920 :   return getNode(Opcode, DL, VTList, Ops);
    6544             : }
    6545             : 
    6546      135414 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6547             :                               SDValue N1, SDValue N2, SDValue N3) {
    6548      135414 :   SDValue Ops[] = { N1, N2, N3 };
    6549      135414 :   return getNode(Opcode, DL, VTList, Ops);
    6550             : }
    6551             : 
    6552         767 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6553             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
    6554         767 :   SDValue Ops[] = { N1, N2, N3, N4 };
    6555         767 :   return getNode(Opcode, DL, VTList, Ops);
    6556             : }
    6557             : 
    6558         332 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6559             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
    6560             :                               SDValue N5) {
    6561         332 :   SDValue Ops[] = { N1, N2, N3, N4, N5 };
    6562         332 :   return getNode(Opcode, DL, VTList, Ops);
    6563             : }
    6564             : 
    6565    30114966 : SDVTList SelectionDAG::getVTList(EVT VT) {
    6566    30114966 :   return makeVTList(SDNode::getValueTypeList(VT), 1);
    6567             : }
    6568             : 
    6569     3822227 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
    6570             :   FoldingSetNodeID ID;
    6571     3822227 :   ID.AddInteger(2U);
    6572     3822227 :   ID.AddInteger(VT1.getRawBits());
    6573     3822227 :   ID.AddInteger(VT2.getRawBits());
    6574             : 
    6575     3822227 :   void *IP = nullptr;
    6576             :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6577     3822227 :   if (!Result) {
    6578       71654 :     EVT *Array = Allocator.Allocate<EVT>(2);
    6579       71654 :     Array[0] = VT1;
    6580       71654 :     Array[1] = VT2;
    6581      143308 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
    6582       71654 :     VTListMap.InsertNode(Result, IP);
    6583             :   }
    6584     7644454 :   return Result->getSDVTList();
    6585             : }
    6586             : 
    6587       43213 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
    6588             :   FoldingSetNodeID ID;
    6589       43213 :   ID.AddInteger(3U);
    6590       43213 :   ID.AddInteger(VT1.getRawBits());
    6591       43213 :   ID.AddInteger(VT2.getRawBits());
    6592       43213 :   ID.AddInteger(VT3.getRawBits());
    6593             : 
    6594       43213 :   void *IP = nullptr;
    6595             :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6596       43213 :   if (!Result) {
    6597        4690 :     EVT *Array = Allocator.Allocate<EVT>(3);
    6598        4690 :     Array[0] = VT1;
    6599        4690 :     Array[1] = VT2;
    6600        4690 :     Array[2] = VT3;
    6601        9380 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
    6602        4690 :     VTListMap.InsertNode(Result, IP);
    6603             :   }
    6604       86426 :   return Result->getSDVTList();
    6605             : }
    6606             : 
    6607          65 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
    6608             :   FoldingSetNodeID ID;
    6609          65 :   ID.AddInteger(4U);
    6610          65 :   ID.AddInteger(VT1.getRawBits());
    6611          65 :   ID.AddInteger(VT2.getRawBits());
    6612          65 :   ID.AddInteger(VT3.getRawBits());
    6613          65 :   ID.AddInteger(VT4.getRawBits());
    6614             : 
    6615          65 :   void *IP = nullptr;
    6616             :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6617          65 :   if (!Result) {
    6618          21 :     EVT *Array = Allocator.Allocate<EVT>(4);
    6619          21 :     Array[0] = VT1;
    6620          21 :     Array[1] = VT2;
    6621          21 :     Array[2] = VT3;
    6622          21 :     Array[3] = VT4;
    6623          42 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
    6624          21 :     VTListMap.InsertNode(Result, IP);
    6625             :   }
    6626         130 :   return Result->getSDVTList();
    6627             : }
    6628             : 
    6629     1126662 : SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
    6630     1126662 :   unsigned NumVTs = VTs.size();
    6631             :   FoldingSetNodeID ID;
    6632     1126662 :   ID.AddInteger(NumVTs);
    6633     5343440 :   for (unsigned index = 0; index < NumVTs; index++) {
    6634     4216778 :     ID.AddInteger(VTs[index].getRawBits());
    6635             :   }
    6636             : 
    6637     1126662 :   void *IP = nullptr;
    6638             :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6639     1126662 :   if (!Result) {
    6640       34940 :     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
    6641             :     std::copy(VTs.begin(), VTs.end(), Array);
    6642       69880 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
    6643       34940 :     VTListMap.InsertNode(Result, IP);
    6644             :   }
    6645     2253324 :   return Result->getSDVTList();
    6646             : }
    6647             : 
    6648             : 
    6649             : /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
    6650             : /// specified operands.  If the resultant node already exists in the DAG,
    6651             : /// this does not modify the specified node, instead it returns the node that
    6652             : /// already exists.  If the resultant node does not exist in the DAG, the
    6653             : /// input node is returned.  As a degenerate case, if you specify the same
    6654             : /// input operands as the node already has, the input node is returned.
    6655        2256 : SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
    6656             :   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
    6657             : 
    6658             :   // Check to see if there is no change.
    6659        2256 :   if (Op == N->getOperand(0)) return N;
    6660             : 
    6661             :   // See if the modified node already exists.
    6662        2256 :   void *InsertPos = nullptr;
    6663        2256 :   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
    6664             :     return Existing;
    6665             : 
    6666             :   // Nope it doesn't.  Remove the node from its current place in the maps.
    6667        2256 :   if (InsertPos)
    6668        2256 :     if (!RemoveNodeFromCSEMaps(N))
    6669           0 :       InsertPos = nullptr;
    6670             : 
    6671             :   // Now we update the operands.
    6672        2256 :   N->OperandList[0].set(Op);
    6673             : 
    6674             :   // If this gets put into a CSE map, add it.
    6675        2256 :   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
    6676             :   return N;
    6677             : }
    6678             : 
    6679       56416 : SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
    6680             :   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
    6681             : 
    6682             :   // Check to see if there is no change.
    6683       56416 :   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
    6684             :     return N;   // No operands changed, just return the input node.
    6685             : 
    6686             :   // See if the modified node already exists.
    6687       56416 :   void *InsertPos = nullptr;
    6688       56416 :   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
    6689             :     return Existing;
    6690             : 
    6691             :   // Nope it doesn't.  Remove the node from its current place in the maps.
    6692       55717 :   if (InsertPos)
    6693       55717 :     if (!RemoveNodeFromCSEMaps(N))
    6694           0 :       InsertPos = nullptr;
    6695             : 
    6696             :   // Now we update the operands.
    6697       55717 :   if (N->OperandList[0] != Op1)
    6698       13161 :     N->OperandList[0].set(Op1);
    6699       55717 :   if (N->OperandList[1] != Op2)
    6700       48424 :     N->OperandList[1].set(Op2);
    6701             : 
    6702             :   // If this gets put into a CSE map, add it.
    6703       55717 :   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
    6704             :   return N;
    6705             : }
    6706             : 
    6707       46018 : SDNode *SelectionDAG::
    6708             : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
    6709       46018 :   SDValue Ops[] = { Op1, Op2, Op3 };
    6710       46018 :   return UpdateNodeOperands(N, Ops);
    6711             : }
    6712             : 
    6713         537 : SDNode *SelectionDAG::
    6714             : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
    6715             :                    SDValue Op3, SDValue Op4) {
    6716         537 :   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
    6717         537 :   return UpdateNodeOperands(N, Ops);
    6718             : }
    6719             : 
    6720        1962 : SDNode *SelectionDAG::
    6721             : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
    6722             :                    SDValue Op3, SDValue Op4, SDValue Op5) {
    6723        1962 :   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
    6724        1962 :   return UpdateNodeOperands(N, Ops);
    6725             : }
    6726             : 
    6727     3798349 : SDNode *SelectionDAG::
    6728             : UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
    6729     3798349 :   unsigned NumOps = Ops.size();
    6730             :   assert(N->getNumOperands() == NumOps &&
    6731             :          "Update with wrong number of operands");
    6732             : 
    6733             :   // If no operands changed just return the input node.
    6734     7596698 :   if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
    6735             :     return N;
    6736             : 
    6737             :   // See if the modified node already exists.
    6738      109082 :   void *InsertPos = nullptr;
    6739      109082 :   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
    6740             :     return Existing;
    6741             : 
    6742             :   // Nope it doesn't.  Remove the node from its current place in the maps.
    6743      108896 :   if (InsertPos)
    6744       98200 :     if (!RemoveNodeFromCSEMaps(N))
    6745           0 :       InsertPos = nullptr;
    6746             : 
    6747             :   // Now we update the operands.
    6748      755388 :   for (unsigned i = 0; i != NumOps; ++i)
    6749      646492 :     if (N->OperandList[i] != Ops[i])
    6750      155586 :       N->OperandList[i].set(Ops[i]);
    6751             : 
    6752             :   // If this gets put into a CSE map, add it.
    6753      108896 :   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
    6754             :   return N;
    6755             : }
    6756             : 
    6757             : /// DropOperands - Release the operands and set this node to have
    6758             : /// zero operands.
    6759    10134757 : void SDNode::DropOperands() {
    6760             :   // Unlike the code in MorphNodeTo that does this, we don't need to
    6761             :   // watch for dead nodes here.
    6762    51739774 :   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
    6763    15735130 :     SDUse &Use = *I++;
    6764    15735130 :     Use.set(SDValue());
    6765             :   }
    6766    10134757 : }
    6767             : 
    6768             : /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
    6769             : /// machine opcode.
    6770             : ///
    6771        6235 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6772             :                                    EVT VT) {
    6773        6235 :   SDVTList VTs = getVTList(VT);
    6774        6235 :   return SelectNodeTo(N, MachineOpc, VTs, None);
    6775             : }
    6776             : 
    6777         179 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6778             :                                    EVT VT, SDValue Op1) {
    6779         179 :   SDVTList VTs = getVTList(VT);
    6780         179 :   SDValue Ops[] = { Op1 };
    6781         179 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6782             : }
    6783             : 
    6784        2109 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6785             :                                    EVT VT, SDValue Op1,
    6786             :                                    SDValue Op2) {
    6787        2109 :   SDVTList VTs = getVTList(VT);
    6788        2109 :   SDValue Ops[] = { Op1, Op2 };
    6789        2109 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6790             : }
    6791             : 
    6792          62 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6793             :                                    EVT VT, SDValue Op1,
    6794             :                                    SDValue Op2, SDValue Op3) {
    6795          62 :   SDVTList VTs = getVTList(VT);
    6796          62 :   SDValue Ops[] = { Op1, Op2, Op3 };
    6797          62 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6798             : }
    6799             : 
    6800        4023 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6801             :                                    EVT VT, ArrayRef<SDValue> Ops) {
    6802        4023 :   SDVTList VTs = getVTList(VT);
    6803        4023 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6804             : }
    6805             : 
    6806           2 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6807             :                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
    6808           2 :   SDVTList VTs = getVTList(VT1, VT2);
    6809           2 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6810             : }
    6811             : 
    6812           4 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6813             :                                    EVT VT1, EVT VT2) {
    6814           4 :   SDVTList VTs = getVTList(VT1, VT2);
    6815           4 :   return SelectNodeTo(N, MachineOpc, VTs, None);
    6816             : }
    6817             : 
    6818           5 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6819             :                                    EVT VT1, EVT VT2, EVT VT3,
    6820             :                                    ArrayRef<SDValue> Ops) {
    6821           5 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    6822           5 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6823             : }
    6824             : 
    6825           0 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6826             :                                    EVT VT1, EVT VT2,
    6827             :                                    SDValue Op1, SDValue Op2) {
    6828           0 :   SDVTList VTs = getVTList(VT1, VT2);
    6829           0 :   SDValue Ops[] = { Op1, Op2 };
    6830           0 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6831             : }
    6832             : 
    6833       31084 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6834             :                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
    6835       31084 :   SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
    6836             :   // Reset the NodeID to -1.
    6837             :   New->setNodeId(-1);
    6838       31084 :   if (New != N) {
    6839          75 :     ReplaceAllUsesWith(N, New);
    6840          75 :     RemoveDeadNode(N);
    6841             :   }
    6842       31084 :   return New;
    6843             : }
    6844             : 
    6845             : /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
    6846             : /// the line number information on the merged node since it is not possible to
    6847             : /// preserve the information that operation is associated with multiple lines.
    6848             : /// This will make the debugger working better at -O0, were there is a higher
    6849             : /// probability having other instructions associated with that line.
    6850             : ///
    6851             : /// For IROrder, we keep the smaller of the two
    6852       93206 : SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
    6853             :   DebugLoc NLoc = N->getDebugLoc();
    6854       93219 :   if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
    6855           0 :     N->setDebugLoc(DebugLoc());
    6856             :   }
    6857      279618 :   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
    6858             :   N->setIROrder(Order);
    6859       93206 :   return N;
    6860             : }
    6861             : 
    6862             : /// MorphNodeTo - This *mutates* the specified node to have the specified
    6863             : /// return type, opcode, and operands.
    6864             : ///
    6865             : /// Note that MorphNodeTo returns the resultant node.  If there is already a
    6866             : /// node of the specified opcode and operands, it returns that node instead of
    6867             : /// the current one.  Note that the SDLoc need not be the same.
    6868             : ///
    6869             : /// Using MorphNodeTo is faster than creating a new node and swapping it in
    6870             : /// with ReplaceAllUsesWith both because it often avoids allocating a new
    6871             : /// node, and because it doesn't require CSE recalculation for any of
    6872             : /// the node's users.
    6873             : ///
    6874             : /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
    6875             : /// As a consequence it isn't appropriate to use from within the DAG combiner or
    6876             : /// the legalizer which maintain worklists that would need to be updated when
    6877             : /// deleting things.
    6878     2803369 : SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
    6879             :                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
    6880             :   // If an identical node already exists, use it.
    6881     2803369 :   void *IP = nullptr;
    6882     2803369 :   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
    6883             :     FoldingSetNodeID ID;
    6884     2192241 :     AddNodeIDNode(ID, Opc, VTs, Ops);
    6885     4384482 :     if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
    6886       32184 :       return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
    6887             :   }
    6888             : 
    6889     2787277 :   if (!RemoveNodeFromCSEMaps(N))
    6890      590549 :     IP = nullptr;
    6891             : 
    6892             :   // Start the morphing.
    6893     2787277 :   N->NodeType = Opc;
    6894     2787277 :   N->ValueList = VTs.VTs;
    6895     2787277 :   N->NumValues = VTs.NumVTs;
    6896             : 
    6897             :   // Clear the operands list, updating used nodes to remove this from their
    6898             :   // use list.  Keep track of any operands that become dead as a result.
    6899             :   SmallPtrSet<SDNode*, 16> DeadNodeSet;
    6900    13744718 :   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
    6901     8170164 :     SDUse &Use = *I++;
    6902             :     SDNode *Used = Use.getNode();
    6903     8170164 :     Use.set(SDValue());
    6904     8170165 :     if (Used->use_empty())
    6905     3436642 :       DeadNodeSet.insert(Used);
    6906             :   }
    6907             : 
    6908             :   // For MachineNode, initialize the memory references information.
    6909             :   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
    6910             :     MN->setMemRefs(nullptr, nullptr);
    6911             : 
    6912             :   // Swap for an appropriately sized array from the recycler.
    6913     2787277 :   removeOperands(N);
    6914     2787277 :   createOperands(N, Ops);
    6915             : 
    6916             :   // Delete any nodes that are still dead after adding the uses for the
    6917             :   // new operands.
    6918     2787277 :   if (!DeadNodeSet.empty()) {
    6919             :     SmallVector<SDNode *, 16> DeadNodes;
    6920     5776700 :     for (SDNode *N : DeadNodeSet)
    6921     3436642 :       if (N->use_empty())
    6922     1208333 :         DeadNodes.push_back(N);
    6923     2340058 :     RemoveDeadNodes(DeadNodes);
    6924             :   }
    6925             : 
    6926     2787277 :   if (IP)
    6927             :     CSEMap.InsertNode(N, IP);   // Memoize the new node.
    6928             :   return N;
    6929             : }
    6930             : 
    6931          40 : SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
    6932          40 :   unsigned OrigOpc = Node->getOpcode();
    6933             :   unsigned NewOpc;
    6934             :   bool IsUnary = false;
    6935             :   bool IsTernary = false;
    6936          40 :   switch (OrigOpc) {
    6937           0 :   default:
    6938           0 :     llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
    6939             :   case ISD::STRICT_FADD: NewOpc = ISD::FADD; break;
    6940           6 :   case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break;
    6941           2 :   case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break;
    6942           2 :   case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break;
    6943           0 :   case ISD::STRICT_FREM: NewOpc = ISD::FREM; break;
    6944           4 :   case ISD::STRICT_FMA: NewOpc = ISD::FMA; IsTernary = true; break;
    6945           2 :   case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; IsUnary = true; break;
    6946           2 :   case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break;
    6947           2 :   case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break;
    6948           2 :   case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; IsUnary = true; break;
    6949           2 :   case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; IsUnary = true; break;
    6950           2 :   case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; IsUnary = true; break;
    6951           2 :   case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; IsUnary = true; break;
    6952           2 :   case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; IsUnary = true; break;
    6953           2 :   case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; IsUnary = true; break;
    6954           2 :   case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; IsUnary = true; break;
    6955           2 :   case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; IsUnary = true; break;
    6956           2 :   case ISD::STRICT_FNEARBYINT:
    6957             :     NewOpc = ISD::FNEARBYINT;
    6958             :     IsUnary = true;
    6959           2 :     break;
    6960             :   }
    6961             : 
    6962             :   // We're taking this node out of the chain, so we need to re-link things.
    6963          40 :   SDValue InputChain = Node->getOperand(0);
    6964             :   SDValue OutputChain = SDValue(Node, 1);
    6965          40 :   ReplaceAllUsesOfValueWith(OutputChain, InputChain);
    6966             : 
    6967          80 :   SDVTList VTs = getVTList(Node->getOperand(1).getValueType());
    6968             :   SDNode *Res = nullptr;
    6969          40 :   if (IsUnary)
    6970          40 :     Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) });
    6971          20 :   else if (IsTernary)
    6972           8 :     Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
    6973             :                                            Node->getOperand(2),
    6974             :                                            Node->getOperand(3)});
    6975             :   else
    6976          32 :     Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
    6977             :                                            Node->getOperand(2) });
    6978             : 
    6979             :   // MorphNodeTo can operate in two ways: if an existing node with the
    6980             :   // specified operands exists, it can just return it.  Otherwise, it
    6981             :   // updates the node in place to have the requested operands.
    6982          40 :   if (Res == Node) {
    6983             :     // If we updated the node in place, reset the node ID.  To the isel,
    6984             :     // this should be just like a newly allocated machine node.
    6985             :     Res->setNodeId(-1);
    6986             :   } else {
    6987           0 :     ReplaceAllUsesWith(Node, Res);
    6988           0 :     RemoveDeadNode(Node);
    6989             :   }
    6990             : 
    6991          40 :   return Res;
    6992             : }
    6993             : 
    6994             : /// getMachineNode - These are used for target selectors to create a new node
    6995             : /// with specified return type(s), MachineInstr opcode, and operands.
    6996             : ///
    6997             : /// Note that getMachineNode returns the resultant node.  If there is already a
    6998             : /// node of the specified opcode and operands, it returns that node instead of
    6999             : /// the current one.
    7000        2174 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7001             :                                             EVT VT) {
    7002        2174 :   SDVTList VTs = getVTList(VT);
    7003        2174 :   return getMachineNode(Opcode, dl, VTs, None);
    7004             : }
    7005             : 
    7006       52819 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7007             :                                             EVT VT, SDValue Op1) {
    7008       52819 :   SDVTList VTs = getVTList(VT);
    7009       52819 :   SDValue Ops[] = { Op1 };
    7010       52819 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7011             : }
    7012             : 
    7013       58500 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7014             :                                             EVT VT, SDValue Op1, SDValue Op2) {
    7015       58500 :   SDVTList VTs = getVTList(VT);
    7016       58500 :   SDValue Ops[] = { Op1, Op2 };
    7017       58500 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7018             : }
    7019             : 
    7020        4220 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7021             :                                             EVT VT, SDValue Op1, SDValue Op2,
    7022             :                                             SDValue Op3) {
    7023        4220 :   SDVTList VTs = getVTList(VT);
    7024        4220 :   SDValue Ops[] = { Op1, Op2, Op3 };
    7025        4220 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7026             : }
    7027             : 
    7028       38116 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7029             :                                             EVT VT, ArrayRef<SDValue> Ops) {
    7030       38116 :   SDVTList VTs = getVTList(VT);
    7031       38116 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7032             : }
    7033             : 
    7034        7949 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7035             :                                             EVT VT1, EVT VT2, SDValue Op1,
    7036             :                                             SDValue Op2) {
    7037        7949 :   SDVTList VTs = getVTList(VT1, VT2);
    7038        7949 :   SDValue Ops[] = { Op1, Op2 };
    7039        7949 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7040             : }
    7041             : 
    7042          96 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7043             :                                             EVT VT1, EVT VT2, SDValue Op1,
    7044             :                                             SDValue Op2, SDValue Op3) {
    7045          96 :   SDVTList VTs = getVTList(VT1, VT2);
    7046          96 :   SDValue Ops[] = { Op1, Op2, Op3 };
    7047          96 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7048             : }
    7049             : 
    7050      131883 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7051             :                                             EVT VT1, EVT VT2,
    7052             :                                             ArrayRef<SDValue> Ops) {
    7053      131883 :   SDVTList VTs = getVTList(VT1, VT2);
    7054      131883 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7055             : }
    7056             : 
    7057          16 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7058             :                                             EVT VT1, EVT VT2, EVT VT3,
    7059             :                                             SDValue Op1, SDValue Op2) {
    7060          16 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    7061          16 :   SDValue Ops[] = { Op1, Op2 };
    7062          16 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7063             : }
    7064             : 
    7065          51 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7066             :                                             EVT VT1, EVT VT2, EVT VT3,
    7067             :                                             SDValue Op1, SDValue Op2,
    7068             :                                             SDValue Op3) {
    7069          51 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    7070          51 :   SDValue Ops[] = { Op1, Op2, Op3 };
    7071          51 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7072             : }
    7073             : 
    7074         378 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7075             :                                             EVT VT1, EVT VT2, EVT VT3,
    7076             :                                             ArrayRef<SDValue> Ops) {
    7077         378 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    7078         378 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7079             : }
    7080             : 
    7081        7945 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    7082             :                                             ArrayRef<EVT> ResultTys,
    7083             :                                             ArrayRef<SDValue> Ops) {
    7084        7945 :   SDVTList VTs = getVTList(ResultTys);
    7085        7945 :   return getMachineNode(Opcode, dl, VTs, Ops);
    7086             : }
    7087             : 
    7088      450203 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
    7089             :                                             SDVTList VTs,
    7090             :                                             ArrayRef<SDValue> Ops) {
    7091      450203 :   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
    7092             :   MachineSDNode *N;
    7093      450203 :   void *IP = nullptr;
    7094             : 
    7095             :   if (DoCSE) {
    7096             :     FoldingSetNodeID ID;
    7097      437252 :     AddNodeIDNode(ID, ~Opcode, VTs, Ops);
    7098      437252 :     IP = nullptr;
    7099      437252 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
    7100       77114 :       return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
    7101             :     }
    7102             :   }
    7103             : 
    7104             :   // Allocate a new MachineSDNode.
    7105      746178 :   N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    7106      373089 :   createOperands(N, Ops);
    7107             : 
    7108      373089 :   if (DoCSE)
    7109      360138 :     CSEMap.InsertNode(N, IP);
    7110             : 
    7111      373089 :   InsertNode(N);
    7112      373089 :   return N;
    7113             : }
    7114             : 
    7115             : /// getTargetExtractSubreg - A convenience function for creating
    7116             : /// TargetOpcode::EXTRACT_SUBREG nodes.
    7117       43216 : SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
    7118             :                                              SDValue Operand) {
    7119       86432 :   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
    7120             :   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
    7121       43216 :                                   VT, Operand, SRIdxVal);
    7122       43216 :   return SDValue(Subreg, 0);
    7123             : }
    7124             : 
    7125             : /// getTargetInsertSubreg - A convenience function for creating
    7126             : /// TargetOpcode::INSERT_SUBREG nodes.
    7127         641 : SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
    7128             :                                             SDValue Operand, SDValue Subreg) {
    7129        1282 :   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
    7130             :   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
    7131         641 :                                   VT, Operand, Subreg, SRIdxVal);
    7132         641 :   return SDValue(Result, 0);
    7133             : }
    7134             : 
    7135             : /// getNodeIfExists - Get the specified node if it's already available, or
    7136             : /// else return NULL.
    7137      628859 : SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
    7138             :                                       ArrayRef<SDValue> Ops,
    7139             :                                       const SDNodeFlags Flags) {
    7140      628859 :   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
    7141             :     FoldingSetNodeID ID;
    7142      628859 :     AddNodeIDNode(ID, Opcode, VTList, Ops);
    7143      628859 :     void *IP = nullptr;
    7144     1257718 :     if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
    7145          25 :       E->intersectFlagsWith(Flags);
    7146             :       return E;
    7147             :     }
    7148             :   }
    7149             :   return nullptr;
    7150             : }
    7151             : 
    7152             : /// getDbgValue - Creates a SDDbgValue node.
    7153             : ///
    7154             : /// SDNode
    7155       14787 : SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
    7156             :                                       SDNode *N, unsigned R, bool IsIndirect,
    7157             :                                       const DebugLoc &DL, unsigned O) {
    7158             :   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
    7159             :          "Expected inlined-at fields to agree");
    7160       14787 :   return new (DbgInfo->getAlloc())
    7161       14787 :       SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O);
    7162             : }
    7163             : 
    7164             : /// Constant
    7165        9095 : SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
    7166             :                                               DIExpression *Expr,
    7167             :                                               const Value *C,
    7168             :                                               const DebugLoc &DL, unsigned O) {
    7169             :   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
    7170             :          "Expected inlined-at fields to agree");
    7171       18190 :   return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O);
    7172             : }
    7173             : 
    7174             : /// FrameIndex
    7175        4003 : SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
    7176             :                                                 DIExpression *Expr, unsigned FI,
    7177             :                                                 const DebugLoc &DL,
    7178             :                                                 unsigned O) {
    7179             :   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
    7180             :          "Expected inlined-at fields to agree");
    7181        8006 :   return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, DL, O);
    7182             : }
    7183             : 
    7184     6440648 : void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
    7185             :                                      unsigned OffsetInBits, unsigned SizeInBits,
    7186             :                                      bool InvalidateDbg) {
    7187             :   SDNode *FromNode = From.getNode();
    7188             :   SDNode *ToNode = To.getNode();
    7189             :   assert(FromNode && ToNode && "Can't modify dbg values");
    7190             : 
    7191             :   // PR35338
    7192             :   // TODO: assert(From != To && "Redundant dbg value transfer");
    7193             :   // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
    7194     6440055 :   if (From == To || FromNode == ToNode)
    7195     6439400 :     return;
    7196             : 
    7197     5593561 :   if (!FromNode->getHasDebugValue())
    7198             :     return;
    7199             : 
    7200             :   SmallVector<SDDbgValue *, 2> ClonedDVs;
    7201        7050 :   for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
    7202        2744 :     if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated())
    7203         984 :       continue;
    7204             : 
    7205             :     // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
    7206             : 
    7207             :     // Just transfer the dbg value attached to From.
    7208        1855 :     if (Dbg->getResNo() != From.getResNo())
    7209          45 :       continue;
    7210             : 
    7211        1765 :     DIVariable *Var = Dbg->getVariable();
    7212        1765 :     auto *Expr = Dbg->getExpression();
    7213             :     // If a fragment is requested, update the expression.
    7214        1765 :     if (SizeInBits) {
    7215             :       // When splitting a larger (e.g., sign-extended) value whose
    7216             :       // lower bits are described with an SDDbgValue, do not attempt
    7217             :       // to transfer the SDDbgValue to the upper bits.
    7218          34 :       if (auto FI = Expr->getFragmentInfo())
    7219          10 :         if (OffsetInBits + SizeInBits > FI->SizeInBits)
    7220             :           continue;
    7221             :       auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
    7222          33 :                                                              SizeInBits);
    7223          33 :       if (!Fragment)
    7224             :         continue;
    7225          29 :       Expr = *Fragment;
    7226             :     }
    7227             :     // Clone the SDDbgValue and move it to To.
    7228             :     SDDbgValue *Clone =
    7229        3520 :         getDbgValue(Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(),
    7230        3520 :                     Dbg->getDebugLoc(), Dbg->getOrder());
    7231        1760 :     ClonedDVs.push_back(Clone);
    7232             : 
    7233        1760 :     if (InvalidateDbg)
    7234             :       Dbg->setIsInvalidated();
    7235             :   }
    7236             : 
    7237        4768 :   for (SDDbgValue *Dbg : ClonedDVs)
    7238        1760 :     AddDbgValue(Dbg, ToNode, false);
    7239             : }
    7240             : 
    7241     1936794 : void SelectionDAG::salvageDebugInfo(SDNode &N) {
    7242     1936794 :   if (!N.getHasDebugValue())
    7243     1935249 :     return;
    7244             : 
    7245             :   SmallVector<SDDbgValue *, 2> ClonedDVs;
    7246       11790 :   for (auto DV : GetDbgValues(&N)) {
    7247        4350 :     if (DV->isInvalidated())
    7248           0 :       continue;
    7249        8700 :     switch (N.getOpcode()) {
    7250             :     default:
    7251        3023 :       break;
    7252        1327 :     case ISD::ADD:
    7253        1327 :       SDValue N0 = N.getOperand(0);
    7254        1327 :       SDValue N1 = N.getOperand(1);
    7255        2654 :       if (!isConstantIntBuildVectorOrConstantInt(N0) &&
    7256        1327 :           isConstantIntBuildVectorOrConstantInt(N1)) {
    7257             :         uint64_t Offset = N.getConstantOperandVal(1);
    7258             :         // Rewrite an ADD constant node into a DIExpression. Since we are
    7259             :         // performing arithmetic to compute the variable's *value* in the
    7260             :         // DIExpression, we need to mark the expression with a
    7261             :         // DW_OP_stack_value.
    7262        1047 :         auto *DIExpr = DV->getExpression();
    7263        1047 :         DIExpr = DIExpression::prepend(DIExpr, DIExpression::NoDeref, Offset,
    7264             :                                        DIExpression::NoDeref,
    7265             :                                        DIExpression::WithStackValue);
    7266             :         SDDbgValue *Clone =
    7267        2094 :             getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(),
    7268        3141 :                         DV->isIndirect(), DV->getDebugLoc(), DV->getOrder());
    7269        1047 :         ClonedDVs.push_back(Clone);
    7270             :         DV->setIsInvalidated();
    7271             :         DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
    7272             :               dbgs() << " into " << *DIExpr << '\n');
    7273             :       }
    7274             :     }
    7275             :   }
    7276             : 
    7277        3639 :   for (SDDbgValue *Dbg : ClonedDVs)
    7278        1047 :     AddDbgValue(Dbg, Dbg->getSDNode(), false);
    7279             : }
    7280             : 
    7281             : namespace {
    7282             : 
    7283             : /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
    7284             : /// pointed to by a use iterator is deleted, increment the use iterator
    7285             : /// so that it doesn't dangle.
    7286             : ///
    7287           0 : class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
    7288             :   SDNode::use_iterator &UI;
    7289             :   SDNode::use_iterator &UE;
    7290             : 
    7291       23280 :   void NodeDeleted(SDNode *N, SDNode *E) override {
    7292             :     // Increment the iterator as needed.
    7293       47561 :     while (UI != UE && N == *UI)
    7294             :       ++UI;
    7295       23280 :   }
    7296             : 
    7297             : public:
    7298             :   RAUWUpdateListener(SelectionDAG &d,
    7299             :                      SDNode::use_iterator &ui,
    7300             :                      SDNode::use_iterator &ue)
    7301    10641640 :     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
    7302             : };
    7303             : 
    7304             : } // end anonymous namespace
    7305             : 
    7306             : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
    7307             : /// This can cause recursive merging of nodes in the DAG.
    7308             : ///
    7309             : /// This version assumes From has a single result value.
    7310             : ///
    7311     2062999 : void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
    7312             :   SDNode *From = FromN.getNode();
    7313             :   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
    7314             :          "Cannot replace with this method!");
    7315             :   assert(From != To.getNode() && "Cannot replace uses of with self");
    7316             : 
    7317             :   // Preserve Debug Values
    7318     2062999 :   transferDbgValues(FromN, To);
    7319             : 
    7320             :   // Iterate over all the existing uses of From. New uses will be added
    7321             :   // to the beginning of the use list, which we avoid visiting.
    7322             :   // This specifically avoids visiting uses of From that arise while the
    7323             :   // replacement is happening, because any such uses would be the result
    7324             :   // of CSE: If an existing node looks like From after one of its operands
    7325             :   // is replaced by To, we don't want to replace of all its users with To
    7326             :   // too. See PR3018 for more info.
    7327     2062999 :   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
    7328             :   RAUWUpdateListener Listener(*this, UI, UE);
    7329     7068601 :   while (UI != UE) {
    7330             :     SDNode *User = *UI;
    7331             : 
    7332             :     // This node is about to morph, remove its old self from the CSE maps.
    7333     2502801 :     RemoveNodeFromCSEMaps(User);
    7334             : 
    7335             :     // A user can appear in a use list multiple times, and when this
    7336             :     // happens the uses are usually next to each other in the list.
    7337             :     // To help reduce the number of CSE recomputations, process all
    7338             :     // the uses of this user that we can find this way.
    7339             :     do {
    7340             :       SDUse &Use = UI.getUse();
    7341             :       ++UI;
    7342     2510743 :       Use.set(To);
    7343     3000568 :     } while (UI != UE && *UI == User);
    7344             : 
    7345             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7346             :     // already exists there, recursively merge the results together.
    7347     2502801 :     AddModifiedNodeToCSEMaps(User);
    7348             :   }
    7349             : 
    7350             :   // If we just RAUW'd the root, take note.
    7351             :   if (FromN == getRoot())
    7352        1498 :     setRoot(To);
    7353     2062999 : }
    7354             : 
    7355             : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
    7356             : /// This can cause recursive merging of nodes in the DAG.
    7357             : ///
    7358             : /// This version assumes that for each value of From, there is a
    7359             : /// corresponding value in To in the same position with the same type.
    7360             : ///
    7361     1425259 : void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
    7362             : #ifndef NDEBUG
    7363             :   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
    7364             :     assert((!From->hasAnyUseOfValue(i) ||
    7365             :             From->getValueType(i) == To->getValueType(i)) &&
    7366             :            "Cannot use this version of ReplaceAllUsesWith!");
    7367             : #endif
    7368             : 
    7369             :   // Handle the trivial case.
    7370     1425259 :   if (From == To)
    7371           0 :     return;
    7372             : 
    7373             :   // Preserve Debug Info. Only do this if there's a use.
    7374     5883020 :   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
    7375     1516251 :     if (From->hasAnyUseOfValue(i)) {
    7376             :       assert((i < To->getNumValues()) && "Invalid To location");
    7377     1433113 :       transferDbgValues(SDValue(From, i), SDValue(To, i));
    7378             :     }
    7379             : 
    7380             :   // Iterate over just the existing users of From. See the comments in
    7381             :   // the ReplaceAllUsesWith above.
    7382     1425259 :   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
    7383             :   RAUWUpdateListener Listener(*this, UI, UE);
    7384     5397667 :   while (UI != UE) {
    7385             :     SDNode *User = *UI;
    7386             : 
    7387             :     // This node is about to morph, remove its old self from the CSE maps.
    7388     1986204 :     RemoveNodeFromCSEMaps(User);
    7389             : 
    7390             :     // A user can appear in a use list multiple times, and when this
    7391             :     // happens the uses are usually next to each other in the list.
    7392             :     // To help reduce the number of CSE recomputations, process all
    7393             :     // the uses of this user that we can find this way.
    7394             :     do {
    7395             :       SDUse &Use = UI.getUse();
    7396             :       ++UI;
    7397             :       Use.setNode(To);
    7398     2560423 :     } while (UI != UE && *UI == User);
    7399             : 
    7400             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7401             :     // already exists there, recursively merge the results together.
    7402     1986204 :     AddModifiedNodeToCSEMaps(User);
    7403             :   }
    7404             : 
    7405             :   // If we just RAUW'd the root, take note.
    7406     1425259 :   if (From == getRoot().getNode())
    7407       11520 :     setRoot(SDValue(To, getRoot().getResNo()));
    7408             : }
    7409             : 
    7410             : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
    7411             : /// This can cause recursive merging of nodes in the DAG.
    7412             : ///
    7413             : /// This version can replace From with any result values.  To must match the
    7414             : /// number and types of values returned by From.
    7415     1009244 : void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
    7416     2018488 :   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
    7417      575513 :     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
    7418             : 
    7419             :   // Preserve Debug Info.
    7420     2172145 :   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
    7421      869207 :     transferDbgValues(SDValue(From, i), *To);
    7422             : 
    7423             :   // Iterate over just the existing users of From. See the comments in
    7424             :   // the ReplaceAllUsesWith above.
    7425      433731 :   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
    7426             :   RAUWUpdateListener Listener(*this, UI, UE);
    7427     1961571 :   while (UI != UE) {
    7428             :     SDNode *User = *UI;
    7429             : 
    7430             :     // This node is about to morph, remove its old self from the CSE maps.
    7431      763920 :     RemoveNodeFromCSEMaps(User);
    7432             : 
    7433             :     // A user can appear in a use list multiple times, and when this
    7434             :     // happens the uses are usually next to each other in the list.
    7435             :     // To help reduce the number of CSE recomputations, process all
    7436             :     // the uses of this user that we can find this way.
    7437             :     do {
    7438             :       SDUse &Use = UI.getUse();
    7439      768308 :       const SDValue &ToOp = To[Use.getResNo()];
    7440             :       ++UI;
    7441      768308 :       Use.set(ToOp);
    7442     1104557 :     } while (UI != UE && *UI == User);
    7443             : 
    7444             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7445             :     // already exists there, recursively merge the results together.
    7446      763920 :     AddModifiedNodeToCSEMaps(User);
    7447             :   }
    7448             : 
    7449             :   // If we just RAUW'd the root, take note.
    7450      433731 :   if (From == getRoot().getNode())
    7451           9 :     setRoot(SDValue(To[getRoot().getResNo()]));
    7452             : }
    7453             : 
    7454             : /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
    7455             : /// uses of other values produced by From.getNode() alone.  The Deleted
    7456             : /// vector is handled the same way as for ReplaceAllUsesWith.
    7457     2195551 : void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
    7458             :   // Handle the really simple, really trivial case efficiently.
    7459      796720 :   if (From == To) return;
    7460             : 
    7461             :   // Handle the simple, trivial, case efficiently.
    7462     2195169 :   if (From.getNode()->getNumValues() == 1) {
    7463      796338 :     ReplaceAllUsesWith(From, To);
    7464      796338 :     return;
    7465             :   }
    7466             : 
    7467             :   // Preserve Debug Info.
    7468     1398831 :   transferDbgValues(From, To);
    7469             : 
    7470             :   // Iterate over just the existing users of From. See the comments in
    7471             :   // the ReplaceAllUsesWith above.
    7472     1398831 :   SDNode::use_iterator UI = From.getNode()->use_begin(),
    7473             :                        UE = From.getNode()->use_end();
    7474             :   RAUWUpdateListener Listener(*this, UI, UE);
    7475     3520907 :   while (UI != UE) {
    7476             :     SDNode *User = *UI;
    7477             :     bool UserRemovedFromCSEMaps = false;
    7478             : 
    7479             :     // A user can appear in a use list multiple times, and when this
    7480             :     // happens the uses are usually next to each other in the list.
    7481             :     // To help reduce the number of CSE recomputations, process all
    7482             :     // the uses of this user that we can find this way.
    7483             :     do {
    7484             :       SDUse &Use = UI.getUse();
    7485             : 
    7486             :       // Skip uses of different values from the same node.
    7487     3204897 :       if (Use.getResNo() != From.getResNo()) {
    7488             :         ++UI;
    7489     1011616 :         continue;
    7490             :       }
    7491             : 
    7492             :       // If this node hasn't been modified yet, it's still in the CSE maps,
    7493             :       // so remove its old self from the CSE maps.
    7494     1181665 :       if (!UserRemovedFromCSEMaps) {
    7495     1180549 :         RemoveNodeFromCSEMaps(User);
    7496             :         UserRemovedFromCSEMaps = true;
    7497             :       }
    7498             : 
    7499             :       ++UI;
    7500     1181665 :       Use.set(To);
    7501     3041633 :     } while (UI != UE && *UI == User);
    7502             : 
    7503             :     // We are iterating over all uses of the From node, so if a use
    7504             :     // doesn't use the specific value, no changes are made.
    7505     2122076 :     if (!UserRemovedFromCSEMaps)
    7506      941527 :       continue;
    7507             : 
    7508             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7509             :     // already exists there, recursively merge the results together.
    7510     1180549 :     AddModifiedNodeToCSEMaps(User);
    7511             :   }
    7512             : 
    7513             :   // If we just RAUW'd the root, take note.
    7514             :   if (From == getRoot())
    7515       26024 :     setRoot(To);
    7516             : }
    7517             : 
    7518             : namespace {
    7519             : 
    7520             :   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
    7521             :   /// to record information about a use.
    7522             :   struct UseMemo {
    7523             :     SDNode *User;
    7524             :     unsigned Index;
    7525             :     SDUse *Use;
    7526             :   };
    7527             : 
    7528             :   /// operator< - Sort Memos by User.
    7529             :   bool operator<(const UseMemo &L, const UseMemo &R) {
    7530         898 :     return (intptr_t)L.User < (intptr_t)R.User;
    7531             :   }
    7532             : 
    7533             : } // end anonymous namespace
    7534             : 
    7535             : /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
    7536             : /// uses of other values produced by From.getNode() alone.  The same value
    7537             : /// may appear in both the From and To list.  The Deleted vector is
    7538             : /// handled the same way as for ReplaceAllUsesWith.
    7539         607 : void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
    7540             :                                               const SDValue *To,
    7541             :                                               unsigned Num){
    7542             :   // Handle the simple, trivial case efficiently.
    7543         607 :   if (Num == 1)
    7544           0 :     return ReplaceAllUsesOfValueWith(*From, *To);
    7545             : 
    7546         607 :   transferDbgValues(*From, *To);
    7547             : 
    7548             :   // Read up all the uses and make records of them. This helps
    7549             :   // processing new uses that are introduced during the
    7550             :   // replacement process.
    7551             :   SmallVector<UseMemo, 4> Uses;
    7552        3273 :   for (unsigned i = 0; i != Num; ++i) {
    7553        1333 :     unsigned FromResNo = From[i].getResNo();
    7554        1333 :     SDNode *FromNode = From[i].getNode();
    7555        1333 :     for (SDNode::use_iterator UI = FromNode->use_begin(),
    7556        3504 :          E = FromNode->use_end(); UI != E; ++UI) {
    7557             :       SDUse &Use = UI.getUse();
    7558        2171 :       if (Use.getResNo() == FromResNo) {
    7559        1211 :         UseMemo Memo = { *UI, i, &Use };
    7560        1211 :         Uses.push_back(Memo);
    7561             :       }
    7562             :     }
    7563             :   }
    7564             : 
    7565             :   // Sort the uses, so that all the uses from a given User are together.
    7566             :   std::sort(Uses.begin(), Uses.end());
    7567             : 
    7568        1746 :   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
    7569        1746 :        UseIndex != UseIndexEnd; ) {
    7570             :     // We know that this user uses some value of From.  If it is the right
    7571             :     // value, update it.
    7572        2278 :     SDNode *User = Uses[UseIndex].User;
    7573             : 
    7574             :     // This node is about to morph, remove its old self from the CSE maps.
    7575        1139 :     RemoveNodeFromCSEMaps(User);
    7576             : 
    7577             :     // The Uses array is sorted, so all the uses for a given User
    7578             :     // are next to each other in the list.
    7579             :     // To help reduce the number of CSE recomputations, process all
    7580             :     // the uses of this user that we can find this way.
    7581             :     do {
    7582        2422 :       unsigned i = Uses[UseIndex].Index;
    7583        1211 :       SDUse &Use = *Uses[UseIndex].Use;
    7584        1211 :       ++UseIndex;
    7585             : 
    7586        1211 :       Use.set(To[i]);
    7587        1815 :     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
    7588             : 
    7589             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7590             :     // already exists there, recursively merge the results together.
    7591        1139 :     AddModifiedNodeToCSEMaps(User);
    7592             :   }
    7593             : }
    7594             : 
    7595             : /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
    7596             : /// based on their topological order. It returns the maximum id and a vector
    7597             : /// of the SDNodes* in assigned order by reference.
    7598      737291 : unsigned SelectionDAG::AssignTopologicalOrder() {
    7599             :   unsigned DAGSize = 0;
    7600             : 
    7601             :   // SortedPos tracks the progress of the algorithm. Nodes before it are
    7602             :   // sorted, nodes after it are unsorted. When the algorithm completes
    7603             :   // it is at the end of the list.
    7604             :   allnodes_iterator SortedPos = allnodes_begin();
    7605             : 
    7606             :   // Visit all the nodes. Move nodes with no operands to the front of
    7607             :   // the list immediately. Annotate nodes that do have operands with their
    7608             :   // operand count. Before we do this, the Node Id fields of the nodes
    7609             :   // may contain arbitrary values. After, the Node Id fields for nodes
    7610             :   // before SortedPos will contain the topological sort index, and the
    7611             :   // Node Id fields for nodes At SortedPos and after will contain the
    7612             :   // count of outstanding operands.
    7613    22216277 :   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
    7614             :     SDNode *N = &*I++;
    7615    21478986 :     checkForCycles(N, this);
    7616    21478986 :     unsigned Degree = N->getNumOperands();
    7617    21478986 :     if (Degree == 0) {
    7618             :       // A node with no uses, add it to the result array immediately.
    7619     8768629 :       N->setNodeId(DAGSize++);
    7620             :       allnodes_iterator Q(N);
    7621     8768629 :       if (Q != SortedPos)
    7622             :         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
    7623             :       assert(SortedPos != AllNodes.end() && "Overran node list");
    7624             :       ++SortedPos;
    7625             :     } else {
    7626             :       // Temporarily use the Node Id as scratch space for the degree count.
    7627    12710357 :       N->setNodeId(Degree);
    7628             :     }
    7629             :   }
    7630             : 
    7631             :   // Visit all the nodes. As we iterate, move nodes into sorted order,
    7632             :   // such that by the time the end is reached all nodes will be sorted.
    7633    22216277 :   for (SDNode &Node : allnodes()) {
    7634             :     SDNode *N = &Node;
    7635    21478986 :     checkForCycles(N, this);
    7636             :     // N is in sorted position, so all its uses have one less operand
    7637             :     // that needs to be sorted.
    7638    21478986 :     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
    7639    57719390 :          UI != UE; ++UI) {
    7640             :       SDNode *P = *UI;
    7641    36240404 :       unsigned Degree = P->getNodeId();
    7642             :       assert(Degree != 0 && "Invalid node degree");
    7643    36240404 :       --Degree;
    7644    36240404 :       if (Degree == 0) {
    7645             :         // All of P's operands are sorted, so P may sorted now.
    7646    12710357 :         P->setNodeId(DAGSize++);
    7647    12710357 :         if (P->getIterator() != SortedPos)
    7648             :           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
    7649             :         assert(SortedPos != AllNodes.end() && "Overran node list");
    7650             :         ++SortedPos;
    7651             :       } else {
    7652             :         // Update P's outstanding operand count.
    7653    23530047 :         P->setNodeId(Degree);
    7654             :       }
    7655             :     }
    7656    21478986 :     if (Node.getIterator() == SortedPos) {
    7657             : #ifndef NDEBUG
    7658             :       allnodes_iterator I(N);
    7659             :       SDNode *S = &*++I;
    7660             :       dbgs() << "Overran sorted position:\n";
    7661             :       S->dumprFull(this); dbgs() << "\n";
    7662             :       dbgs() << "Checking if this is due to cycles\n";
    7663             :       checkForCycles(this, true);
    7664             : #endif
    7665           0 :       llvm_unreachable(nullptr);
    7666             :     }
    7667             :   }
    7668             : 
    7669             :   assert(SortedPos == AllNodes.end() &&
    7670             :          "Topological sort incomplete!");
    7671             :   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
    7672             :          "First node in topological sort is not the entry token!");
    7673             :   assert(AllNodes.front().getNodeId() == 0 &&
    7674             :          "First node in topological sort has non-zero id!");
    7675             :   assert(AllNodes.front().getNumOperands() == 0 &&
    7676             :          "First node in topological sort has operands!");
    7677             :   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
    7678             :          "Last node in topologic sort has unexpected id!");
    7679             :   assert(AllNodes.back().use_empty() &&
    7680             :          "Last node in topologic sort has users!");
    7681             :   assert(DAGSize == allnodes_size() && "Node count mismatch!");
    7682      737291 :   return DAGSize;
    7683             : }
    7684             : 
    7685             : /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
    7686             : /// value is produced by SD.
    7687       27885 : void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
    7688       27885 :   if (SD) {
    7689             :     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
    7690             :     SD->setHasDebugValue(true);
    7691             :   }
    7692       27885 :   DbgInfo->add(DB, SD, isParameter);
    7693       27885 : }
    7694             : 
    7695        2862 : SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
    7696             :                                                    SDValue NewMemOp) {
    7697             :   assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
    7698             :   // The new memory operation must have the same position as the old load in
    7699             :   // terms of memory dependency. Create a TokenFactor for the old load and new
    7700             :   // memory operation and update uses of the old load's output chain to use that
    7701             :   // TokenFactor.
    7702             :   SDValue OldChain = SDValue(OldLoad, 1);
    7703        2862 :   SDValue NewChain = SDValue(NewMemOp.getNode(), 1);
    7704        2862 :   if (!OldLoad->hasAnyUseOfValue(1))
    7705        2564 :     return NewChain;
    7706             : 
    7707             :   SDValue TokenFactor =
    7708         596 :       getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain);
    7709         298 :   ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
    7710         298 :   UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain);
    7711         298 :   return TokenFactor;
    7712             : }
    7713             : 
    7714             : //===----------------------------------------------------------------------===//
    7715             : //                              SDNode Class
    7716             : //===----------------------------------------------------------------------===//
    7717             : 
    7718     3979954 : bool llvm::isNullConstant(SDValue V) {
    7719             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
    7720     5324282 :   return Const != nullptr && Const->isNullValue();
    7721             : }
    7722             : 
    7723      660822 : bool llvm::isNullFPConstant(SDValue V) {
    7724             :   ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
    7725       91978 :   return Const != nullptr && Const->isZero() && !Const->isNegative();
    7726             : }
    7727             : 
    7728      619858 : bool llvm::isAllOnesConstant(SDValue V) {
    7729             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
    7730      918920 :   return Const != nullptr && Const->isAllOnesValue();
    7731             : }
    7732             : 
    7733      167134 : bool llvm::isOneConstant(SDValue V) {
    7734             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
    7735      192994 :   return Const != nullptr && Const->isOne();
    7736             : }
    7737             : 
    7738      162801 : bool llvm::isBitwiseNot(SDValue V) {
    7739      163869 :   return V.getOpcode() == ISD::XOR && isAllOnesConstant(V.getOperand(1));
    7740             : }
    7741             : 
    7742     6854292 : ConstantSDNode *llvm::isConstOrConstSplat(SDValue N) {
    7743             :   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
    7744             :     return CN;
    7745             : 
    7746             :   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
    7747             :     BitVector UndefElements;
    7748      128543 :     ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
    7749             : 
    7750             :     // BuildVectors can truncate their operands. Ignore that case here.
    7751             :     // FIXME: We blindly ignore splats which include undef which is overly
    7752             :     // pessimistic.
    7753      245292 :     if (CN && UndefElements.none() &&
    7754      477830 :         CN->getValueType(0) == N.getValueType().getScalarType())
    7755             :       return CN;
    7756             :   }
    7757             : 
    7758             :   return nullptr;
    7759             : }
    7760             : 
    7761      163569 : ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N) {
    7762             :   if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
    7763             :     return CN;
    7764             : 
    7765             :   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
    7766             :     BitVector UndefElements;
    7767        3821 :     ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
    7768             : 
    7769        5210 :     if (CN && UndefElements.none())
    7770             :       return CN;
    7771             :   }
    7772             : 
    7773             :   return nullptr;
    7774             : }
    7775             : 
    7776     8219074 : HandleSDNode::~HandleSDNode() {
    7777     4109537 :   DropOperands();
    7778     4109537 : }
    7779             : 
    7780     1438006 : GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
    7781             :                                          const DebugLoc &DL,
    7782             :                                          const GlobalValue *GA, EVT VT,
    7783     1438006 :                                          int64_t o, unsigned char TF)
    7784     4314018 :     : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
    7785     1438006 :   TheGlobal = GA;
    7786     1438006 : }
    7787             : 
    7788         201 : AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
    7789             :                                          EVT VT, unsigned SrcAS,
    7790         201 :                                          unsigned DestAS)
    7791             :     : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
    7792         603 :       SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
    7793             : 
    7794     3820097 : MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
    7795     3820097 :                      SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
    7796    11460291 :     : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
    7797     7640194 :   MemSDNodeBits.IsVolatile = MMO->isVolatile();
    7798     3820097 :   MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
    7799     3820097 :   MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
    7800     3820097 :   MemSDNodeBits.IsInvariant = MMO->isInvariant();
    7801             : 
    7802             :   // We check here that the size of the memory operand fits within the size of
    7803             :   // the MMO. This is because the MMO might indicate only a possible address
    7804             :   // range instead of specifying the affected memory addresses precisely.
    7805             :   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
    7806     3820097 : }
    7807             : 
    7808             : /// Profile - Gather unique data for the node.
    7809             : ///
    7810    34366230 : void SDNode::Profile(FoldingSetNodeID &ID) const {
    7811    34366230 :   AddNodeIDNode(ID, this);
    7812    34366230 : }
    7813             : 
    7814             : namespace {
    7815             : 
    7816       18126 :   struct EVTArray {
    7817             :     std::vector<EVT> VTs;
    7818             : 
    7819       18430 :     EVTArray() {
    7820       18430 :       VTs.reserve(MVT::LAST_VALUETYPE);
    7821     4183610 :       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
    7822     4165180 :         VTs.push_back(MVT((MVT::SimpleValueType)i));
    7823       18430 :     }
    7824             :   };
    7825             : 
    7826             : } // end anonymous namespace
    7827             : 
    7828             : static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
    7829             : static ManagedStatic<EVTArray> SimpleVTArray;
    7830             : static ManagedStatic<sys::SmartMutex<true>> VTMutex;
    7831             : 
    7832             : /// getValueTypeList - Return a pointer to the specified value type.
    7833             : ///
    7834    41428140 : const EVT *SDNode::getValueTypeList(EVT VT) {
    7835    41428140 :   if (VT.isExtended()) {
    7836       79823 :     sys::SmartScopedLock<true> Lock(*VTMutex);
    7837             :     return &(*EVTs->insert(VT).first);
    7838             :   } else {
    7839             :     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
    7840             :            "Value type out of range!");
    7841   124044951 :     return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
    7842             :   }
    7843             : }
    7844             : 
    7845             : /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
    7846             : /// indicated value.  This method ignores uses of other values defined by this
    7847             : /// operation.
    7848     8484955 : bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
    7849             :   assert(Value < getNumValues() && "Bad value!");
    7850             : 
    7851             :   // TODO: Only iterate over uses of a given value of the node
    7852     8484955 :   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
    7853    12765557 :     if (UI.getUse().getResNo() == Value) {
    7854    11200400 :       if (NUses == 0)
    7855             :         return false;
    7856     8484611 :       --NUses;
    7857             :     }
    7858             :   }
    7859             : 
    7860             :   // Found exactly the right number of uses?
    7861     5769166 :   return NUses == 0;
    7862             : }
    7863             : 
    7864             : /// hasAnyUseOfValue - Return true if there are any use of the indicated
    7865             : /// value. This method ignores uses of other values defined by this operation.
    7866     8341652 : bool SDNode::hasAnyUseOfValue(unsigned Value) const {
    7867             :   assert(Value < getNumValues() && "Bad value!");
    7868             : 
    7869     8341652 :   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
    7870     9920868 :     if (UI.getUse().getResNo() == Value)
    7871             :       return true;
    7872             : 
    7873             :   return false;
    7874             : }
    7875             : 
    7876             : /// isOnlyUserOf - Return true if this node is the only use of N.
    7877      109734 : bool SDNode::isOnlyUserOf(const SDNode *N) const {
    7878             :   bool Seen = false;
    7879      109734 :   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
    7880             :     SDNode *User = *I;
    7881      135509 :     if (User == this)
    7882             :       Seen = true;
    7883             :     else
    7884             :       return false;
    7885             :   }
    7886             : 
    7887             :   return Seen;
    7888             : }
    7889             : 
    7890             : /// Return true if the only users of N are contained in Nodes.
    7891       30561 : bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
    7892             :   bool Seen = false;
    7893       30561 :   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
    7894       50170 :     SDNode *User = *I;
    7895       50170 :     if (llvm::any_of(Nodes,
    7896       69596 :                      [&User](const SDNode *Node) { return User == Node; }))
    7897             :       Seen = true;
    7898             :     else
    7899       28058 :       return false;
    7900             :   }
    7901             : 
    7902             :   return Seen;
    7903             : }
    7904             : 
    7905             : /// isOperand - Return true if this node is an operand of N.
    7906      623621 : bool SDValue::isOperandOf(const SDNode *N) const {
    7907     3348375 :   for (const SDValue &Op : N->op_values())
    7908             :     if (*this == Op)
    7909             :       return true;
    7910             :   return false;
    7911             : }
    7912             : 
    7913        2367 : bool SDNode::isOperandOf(const SDNode *N) const {
    7914        6573 :   for (const SDValue &Op : N->op_values())
    7915        5966 :     if (this == Op.getNode())
    7916             :       return true;
    7917             :   return false;
    7918             : }
    7919             : 
    7920             : /// reachesChainWithoutSideEffects - Return true if this operand (which must
    7921             : /// be a chain) reaches the specified operand without crossing any
    7922             : /// side-effecting instructions on any chain path.  In practice, this looks
    7923             : /// through token factors and non-volatile loads.  In order to remain efficient,
    7924             : /// this only looks a couple of nodes in, it does not do an exhaustive search.
    7925             : ///
    7926             : /// Note that we only need to examine chains when we're searching for
    7927             : /// side-effects; SelectionDAG requires that all side-effects are represented
    7928             : /// by chains, even if another operand would force a specific ordering. This
    7929             : /// constraint is necessary to allow transformations like splitting loads.
    7930        3063 : bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
    7931             :                                              unsigned Depth) const {
    7932             :   if (*this == Dest) return true;
    7933             : 
    7934             :   // Don't search too deeply, we just want to be able to see through
    7935             :   // TokenFactor's etc.
    7936         825 :   if (Depth == 0) return false;
    7937             : 
    7938             :   // If this is a token factor, all inputs to the TF happen in parallel.
    7939         790 :   if (getOpcode() == ISD::TokenFactor) {
    7940             :     // First, try a shallow search.
    7941          52 :     if (is_contained((*this)->ops(), Dest)) {
    7942             :       // We found the chain we want as an operand of this TokenFactor.
    7943             :       // Essentially, we reach the chain without side-effects if we could
    7944             :       // serialize the TokenFactor into a simple chain of operations with
    7945             :       // Dest as the last operation. This is automatically true if the
    7946             :       // chain has one use: there are no other ordering constraints.
    7947             :       // If the chain has more than one use, we give up: some other
    7948             :       // use of Dest might force a side-effect between Dest and the current
    7949             :       // node.
    7950          62 :       if (Dest.hasOneUse())
    7951             :         return true;
    7952             :     }
    7953             :     // Next, try a deep search: check whether every operand of the TokenFactor
    7954             :     // reaches Dest.
    7955         120 :     return llvm::all_of((*this)->ops(), [=](SDValue Op) {
    7956          51 :       return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
    7957          93 :     });
    7958             :   }
    7959             : 
    7960             :   // Loads don't have side effects, look through them.
    7961             :   if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
    7962          80 :     if (!Ld->isVolatile())
    7963          80 :       return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
    7964             :   }
    7965             :   return false;
    7966             : }
    7967             : 
    7968        5575 : bool SDNode::hasPredecessor(const SDNode *N) const {
    7969             :   SmallPtrSet<const SDNode *, 32> Visited;
    7970             :   SmallVector<const SDNode *, 16> Worklist;
    7971        5575 :   Worklist.push_back(this);
    7972       11150 :   return hasPredecessorHelper(N, Visited, Worklist);
    7973             : }
    7974             : 
    7975      526128 : void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
    7976      526128 :   this->Flags.intersectWith(Flags);
    7977      526128 : }
    7978             : 
    7979        3860 : SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
    7980             :   assert(N->getNumValues() == 1 &&
    7981             :          "Can't unroll a vector with multiple results!");
    7982             : 
    7983        7720 :   EVT VT = N->getValueType(0);
    7984        3860 :   unsigned NE = VT.getVectorNumElements();
    7985        3860 :   EVT EltVT = VT.getVectorElementType();
    7986             :   SDLoc dl(N);
    7987             : 
    7988             :   SmallVector<SDValue, 8> Scalars;
    7989        7720 :   SmallVector<SDValue, 4> Operands(N->getNumOperands());
    7990             : 
    7991             :   // If ResNE is 0, fully unroll the vector op.
    7992        3860 :   if (ResNE == 0)
    7993             :     ResNE = NE;
    7994          15 :   else if (NE > ResNE)
    7995             :     NE = ResNE;
    7996             : 
    7997             :   unsigned i;
    7998       27572 :   for (i= 0; i != NE; ++i) {
    7999       62160 :     for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
    8000       38448 :       SDValue Operand = N->getOperand(j);
    8001       19224 :       EVT OperandVT = Operand.getValueType();
    8002       19224 :       if (OperandVT.isVector()) {
    8003             :         // A vector operand; extract a single element.
    8004       18784 :         EVT OperandEltVT = OperandVT.getVectorElementType();
    8005       18784 :         Operands[j] =
    8006       37568 :             getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
    8007       75136 :                     getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
    8008             :       } else {
    8009             :         // A scalar operand; just use it as is.
    8010         440 :         Operands[j] = Operand;
    8011             :       }
    8012             :     }
    8013             : 
    8014       23712 :     switch (N->getOpcode()) {
    8015       11294 :     default: {
    8016       11294 :       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
    8017       11294 :                                 N->getFlags()));
    8018       11294 :       break;
    8019             :     }
    8020             :     case ISD::VSELECT:
    8021          52 :       Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
    8022          52 :       break;
    8023             :     case ISD::SHL:
    8024             :     case ISD::SRA:
    8025             :     case ISD::SRL:
    8026             :     case ISD::ROTL:
    8027             :     case ISD::ROTR:
    8028         438 :       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
    8029             :                                getShiftAmountOperand(Operands[0].getValueType(),
    8030        1752 :                                                      Operands[1])));
    8031         438 :       break;
    8032             :     case ISD::SIGN_EXTEND_INREG:
    8033             :     case ISD::FP_ROUND_INREG: {
    8034          72 :       EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
    8035          72 :       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
    8036             :                                 Operands[0],
    8037         216 :                                 getValueType(ExtVT)));
    8038             :     }
    8039             :     }
    8040             :   }
    8041             : 
    8042        3874 :   for (; i < ResNE; ++i)
    8043           7 :     Scalars.push_back(getUNDEF(EltVT));
    8044             : 
    8045        3860 :   EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
    8046        7720 :   return getBuildVector(VecVT, dl, Scalars);
    8047             : }
    8048             : 
    8049        5200 : bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
    8050             :                                                   LoadSDNode *Base,
    8051             :                                                   unsigned Bytes,
    8052             :                                                   int Dist) const {
    8053       10354 :   if (LD->isVolatile() || Base->isVolatile())
    8054             :     return false;
    8055       10218 :   if (LD->isIndexed() || Base->isIndexed())
    8056             :     return false;
    8057             :   if (LD->getChain() != Base->getChain())
    8058             :     return false;
    8059        9352 :   EVT VT = LD->getValueType(0);
    8060        4676 :   if (VT.getSizeInBits() / 8 != Bytes)
    8061             :     return false;
    8062             : 
    8063        4676 :   auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
    8064        4676 :   auto LocDecomp = BaseIndexOffset::match(LD, *this);
    8065             : 
    8066        4676 :   int64_t Offset = 0;
    8067        4676 :   if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
    8068        4427 :     return (Dist * Bytes == Offset);
    8069             :   return false;
    8070             : }
    8071             : 
    8072             : /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
    8073             : /// it cannot be inferred.
    8074     5559020 : unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
    8075             :   // If this is a GlobalAddress + cst, return the alignment.
    8076             :   const GlobalValue *GV;
    8077     5559020 :   int64_t GVOffset = 0;
    8078     5559020 :   if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
    8079     5355108 :     unsigned IdxWidth = getDataLayout().getIndexTypeSizeInBits(GV->getType());
    8080     1789056 :     KnownBits Known(IdxWidth);
    8081     3570072 :     llvm::computeKnownBits(GV, Known, getDataLayout());
    8082     1785036 :     unsigned AlignBits = Known.countMinTrailingZeros();
    8083     3566052 :     unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
    8084     1781016 :     if (Align)
    8085     3562032 :       return MinAlign(Align, GVOffset);
    8086             :   }
    8087             : 
    8088             :   // If this is a direct reference to a stack slot, use information about the
    8089             :   // stack slot's alignment.
    8090             :   int FrameIdx = 1 << 31;
    8091             :   int64_t FrameOffset = 0;
    8092             :   if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
    8093     1098757 :     FrameIdx = FI->getIndex();
    8094     2679247 :   } else if (isBaseWithConstantOffset(Ptr) &&
    8095             :              isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
    8096             :     // Handle FI+Cst
    8097      265584 :     FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
    8098      265584 :     FrameOffset = Ptr.getConstantOperandVal(1);
    8099             :   }
    8100             : 
    8101     1364341 :   if (FrameIdx != (1 << 31)) {
    8102     1364341 :     const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
    8103     2728682 :     unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
    8104     1364341 :                                     FrameOffset);
    8105     1364341 :     return FIInfoAlign;
    8106             :   }
    8107             : 
    8108             :   return 0;
    8109             : }
    8110             : 
    8111             : /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
    8112             : /// which is split (or expanded) into two not necessarily identical pieces.
    8113       94868 : std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
    8114             :   // Currently all types are split in half.
    8115             :   EVT LoVT, HiVT;
    8116       94868 :   if (!VT.isVector())
    8117        1708 :     LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
    8118             :   else
    8119       93160 :     LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
    8120             : 
    8121       94868 :   return std::make_pair(LoVT, HiVT);
    8122             : }
    8123             : 
    8124             : /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
    8125             : /// low/high part.
    8126             : std::pair<SDValue, SDValue>
    8127       13846 : SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
    8128             :                           const EVT &HiVT) {
    8129             :   assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
    8130             :          N.getValueType().getVectorNumElements() &&
    8131             :          "More vector elements requested than available!");
    8132             :   SDValue Lo, Hi;
    8133       13846 :   Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
    8134       55384 :                getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
    8135       13846 :   Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
    8136       13846 :                getConstant(LoVT.getVectorNumElements(), DL,
    8137       55384 :                            TLI->getVectorIdxTy(getDataLayout())));
    8138       13846 :   return std::make_pair(Lo, Hi);
    8139             : }
    8140             : 
    8141       20071 : void SelectionDAG::ExtractVectorElements(SDValue Op,
    8142             :                                          SmallVectorImpl<SDValue> &Args,
    8143             :                                          unsigned Start, unsigned Count) {
    8144       20071 :   EVT VT = Op.getValueType();
    8145       20071 :   if (Count == 0)
    8146        4984 :     Count = VT.getVectorNumElements();
    8147             : 
    8148       20071 :   EVT EltVT = VT.getVectorElementType();
    8149       40142 :   EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
    8150             :   SDLoc SL(Op);
    8151      103427 :   for (unsigned i = Start, e = Start + Count; i != e; ++i) {
    8152      166712 :     Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
    8153      166712 :                            Op, getConstant(i, SL, IdxTy)));
    8154             :   }
    8155       20071 : }
    8156             : 
    8157             : // getAddressSpace - Return the address space this GlobalAddress belongs to.
    8158        7930 : unsigned GlobalAddressSDNode::getAddressSpace() const {
    8159       15860 :   return getGlobal()->getType()->getAddressSpace();
    8160             : }
    8161             : 
    8162       23727 : Type *ConstantPoolSDNode::getType() const {
    8163       23727 :   if (isMachineConstantPoolEntry())
    8164         261 :     return Val.MachineCPVal->getType();
    8165       23466 :   return Val.ConstVal->getType();
    8166             : }
    8167             : 
    8168      314330 : bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
    8169             :                                         unsigned &SplatBitSize,
    8170             :                                         bool &HasAnyUndefs,
    8171             :                                         unsigned MinSplatBits,
    8172             :                                         bool IsBigEndian) const {
    8173      628660 :   EVT VT = getValueType(0);
    8174             :   assert(VT.isVector() && "Expected a vector type");
    8175      314330 :   unsigned VecWidth = VT.getSizeInBits();
    8176      314330 :   if (MinSplatBits > VecWidth)
    8177             :     return false;
    8178             : 
    8179             :   // FIXME: The widths are based on this node's type, but build vectors can
    8180             :   // truncate their operands.
    8181      314330 :   SplatValue = APInt(VecWidth, 0);
    8182      314330 :   SplatUndef = APInt(VecWidth, 0);
    8183             : 
    8184             :   // Get the bits. Bits with undefined values (when the corresponding element
    8185             :   // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
    8186             :   // in SplatValue. If any of the values are not constant, give up and return
    8187             :   // false.
    8188      314330 :   unsigned int NumOps = getNumOperands();
    8189             :   assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
    8190             :   unsigned EltWidth = VT.getScalarSizeInBits();
    8191             : 
    8192     2036638 :   for (unsigned j = 0; j < NumOps; ++j) {
    8193      867949 :     unsigned i = IsBigEndian ? NumOps - 1 - j : j;
    8194     1735898 :     SDValue OpVal = getOperand(i);
    8195      867949 :     unsigned BitPos = j * EltWidth;
    8196             : 
    8197      867949 :     if (OpVal.isUndef())
    8198       13968 :       SplatUndef.setBits(BitPos, BitPos + EltWidth);
    8199             :     else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
    8200     2528016 :       SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
    8201             :     else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
    8202       13542 :       SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
    8203             :     else
    8204             :       return false;
    8205             :   }
    8206             : 
    8207             :   // The build_vector is all constants or undefs. Find the smallest element
    8208             :   // size that splats the vector.
    8209      307535 :   HasAnyUndefs = (SplatUndef != 0);
    8210             : 
    8211             :   // FIXME: This does not work for vectors with elements less than 8 bits.
    8212      629890 :   while (VecWidth > 8) {
    8213      625745 :     unsigned HalfSize = VecWidth / 2;
    8214     1251490 :     APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
    8215      625745 :     APInt LowValue = SplatValue.trunc(HalfSize);
    8216     1251490 :     APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
    8217      625745 :     APInt LowUndef = SplatUndef.trunc(HalfSize);
    8218             : 
    8219             :     // If the two halves do not match (ignoring undef bits), stop here.
    8220     4380215 :     if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
    8221             :         MinSplatBits > HalfSize)
    8222             :       break;
    8223             : 
    8224      322355 :     SplatValue = HighValue | LowValue;
    8225      322355 :     SplatUndef = HighUndef & LowUndef;
    8226             : 
    8227             :     VecWidth = HalfSize;
    8228             :   }
    8229             : 
    8230      307535 :   SplatBitSize = VecWidth;
    8231      307535 :   return true;
    8232             : }
    8233             : 
    8234      357944 : SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
    8235      357944 :   if (UndefElements) {
    8236             :     UndefElements->clear();
    8237      322482 :     UndefElements->resize(getNumOperands());
    8238             :   }
    8239             :   SDValue Splatted;
    8240     3690818 :   for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
    8241     3152146 :     SDValue Op = getOperand(i);
    8242     1576073 :     if (Op.isUndef()) {
    8243       22743 :       if (UndefElements)
    8244             :         (*UndefElements)[i] = true;
    8245     1553330 :     } else if (!Splatted) {
    8246             :       Splatted = Op;
    8247             :     } else if (Splatted != Op) {
    8248       88608 :       return SDValue();
    8249             :     }
    8250             :   }
    8251             : 
    8252      269336 :   if (!Splatted) {
    8253             :     assert(getOperand(0).isUndef() &&
    8254             :            "Can only have a splat without a constant for all undefs.");
    8255          63 :     return getOperand(0);
    8256             :   }
    8257             : 
    8258      269273 :   return Splatted;
    8259             : }
    8260             : 
    8261             : ConstantSDNode *
    8262      142743 : BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
    8263      285486 :   return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
    8264             : }
    8265             : 
    8266             : ConstantFPSDNode *
    8267        4454 : BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
    8268        8908 :   return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
    8269             : }
    8270             : 
    8271             : int32_t
    8272          55 : BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
    8273             :                                                    uint32_t BitWidth) const {
    8274             :   if (ConstantFPSDNode *CN =
    8275          56 :           dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
    8276             :     bool IsExact;
    8277          54 :     APSInt IntVal(BitWidth);
    8278          54 :     const APFloat &APF = CN->getValueAPF();
    8279          54 :     if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
    8280         102 :             APFloat::opOK ||
    8281          48 :         !IsExact)
    8282             :       return -1;
    8283             : 
    8284          48 :     return IntVal.exactLogBase2();
    8285             :   }
    8286             :   return -1;
    8287             : }
    8288             : 
    8289      245687 : bool BuildVectorSDNode::isConstant() const {
    8290     1280248 :   for (const SDValue &Op : op_values()) {
    8291     1076662 :     unsigned Opc = Op.getOpcode();
    8292     1076662 :     if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
    8293             :       return false;
    8294             :   }
    8295             :   return true;
    8296             : }
    8297             : 
    8298       62133 : bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
    8299             :   // Find the first non-undef value in the shuffle mask.
    8300             :   unsigned i, e;
    8301       62133 :   for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
    8302             :     /* search */;
    8303             : 
    8304             :   assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
    8305             : 
    8306             :   // Make sure all remaining elements are either undef or the same as the first
    8307             :   // non-undef value.
    8308      215782 :   for (int Idx = Mask[i]; i != e; ++i)
    8309      204514 :     if (Mask[i] >= 0 && Mask[i] != Idx)
    8310             :       return false;
    8311             :   return true;
    8312             : }
    8313             : 
    8314             : // \brief Returns the SDNode if it is a constant integer BuildVector
    8315             : // or constant integer.
    8316     3878571 : SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
    8317             :   if (isa<ConstantSDNode>(N))
    8318             :     return N.getNode();
    8319     3439578 :   if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
    8320             :     return N.getNode();
    8321             :   // Treat a GlobalAddress supporting constant offset folding as a
    8322             :   // constant integer.
    8323             :   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
    8324      837854 :     if (GA->getOpcode() == ISD::GlobalAddress &&
    8325      418919 :         TLI->isOffsetFoldingLegal(GA))
    8326             :       return GA;
    8327             :   return nullptr;
    8328             : }
    8329             : 
    8330        5752 : SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
    8331             :   if (isa<ConstantFPSDNode>(N))
    8332             :     return N.getNode();
    8333             : 
    8334        5504 :   if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
    8335           4 :     return N.getNode();
    8336             : 
    8337             :   return nullptr;
    8338             : }
    8339             : 
    8340             : #ifndef NDEBUG
    8341             : static void checkForCyclesHelper(const SDNode *N,
    8342             :                                  SmallPtrSetImpl<const SDNode*> &Visited,
    8343             :                                  SmallPtrSetImpl<const SDNode*> &Checked,
    8344             :                                  const llvm::SelectionDAG *DAG) {
    8345             :   // If this node has already been checked, don't check it again.
    8346             :   if (Checked.count(N))
    8347             :     return;
    8348             : 
    8349             :   // If a node has already been visited on this depth-first walk, reject it as
    8350             :   // a cycle.
    8351             :   if (!Visited.insert(N).second) {
    8352             :     errs() << "Detected cycle in SelectionDAG\n";
    8353             :     dbgs() << "Offending node:\n";
    8354             :     N->dumprFull(DAG); dbgs() << "\n";
    8355             :     abort();
    8356             :   }
    8357             : 
    8358             :   for (const SDValue &Op : N->op_values())
    8359             :     checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
    8360             : 
    8361             :   Checked.insert(N);
    8362             :   Visited.erase(N);
    8363             : }
    8364             : #endif
    8365             : 
    8366    67036356 : void llvm::checkForCycles(const llvm::SDNode *N,
    8367             :                           const llvm::SelectionDAG *DAG,
    8368             :                           bool force) {
    8369             : #ifndef NDEBUG
    8370             :   bool check = force;
    8371             : #ifdef EXPENSIVE_CHECKS
    8372             :   check = true;
    8373             : #endif  // EXPENSIVE_CHECKS
    8374             :   if (check) {
    8375             :     assert(N && "Checking nonexistent SDNode");
    8376             :     SmallPtrSet<const SDNode*, 32> visited;
    8377             :     SmallPtrSet<const SDNode*, 32> checked;
    8378             :     checkForCyclesHelper(N, visited, checked, DAG);
    8379             :   }
    8380             : #endif  // !NDEBUG
    8381    67036356 : }
    8382             : 
    8383     4880623 : void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
    8384     4880623 :   checkForCycles(DAG->getRoot().getNode(), DAG, force);
    8385     4880623 : }

Generated by: LCOV version 1.13