LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - SelectionDAG.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 3821 3963 96.4 %
Date: 2017-09-14 15:23:50 Functions: 266 272 97.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This implements the SelectionDAG class.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/CodeGen/SelectionDAG.h"
      15             : #include "SDNodeDbgValue.h"
      16             : #include "llvm/ADT/APFloat.h"
      17             : #include "llvm/ADT/APInt.h"
      18             : #include "llvm/ADT/APSInt.h"
      19             : #include "llvm/ADT/ArrayRef.h"
      20             : #include "llvm/ADT/BitVector.h"
      21             : #include "llvm/ADT/FoldingSet.h"
      22             : #include "llvm/ADT/None.h"
      23             : #include "llvm/ADT/STLExtras.h"
      24             : #include "llvm/ADT/SmallPtrSet.h"
      25             : #include "llvm/ADT/SmallVector.h"
      26             : #include "llvm/ADT/Triple.h"
      27             : #include "llvm/ADT/Twine.h"
      28             : #include "llvm/Analysis/ValueTracking.h"
      29             : #include "llvm/CodeGen/ISDOpcodes.h"
      30             : #include "llvm/CodeGen/MachineBasicBlock.h"
      31             : #include "llvm/CodeGen/MachineConstantPool.h"
      32             : #include "llvm/CodeGen/MachineFrameInfo.h"
      33             : #include "llvm/CodeGen/MachineFunction.h"
      34             : #include "llvm/CodeGen/MachineMemOperand.h"
      35             : #include "llvm/CodeGen/MachineValueType.h"
      36             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      37             : #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
      38             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      39             : #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
      40             : #include "llvm/CodeGen/ValueTypes.h"
      41             : #include "llvm/IR/Constant.h"
      42             : #include "llvm/IR/Constants.h"
      43             : #include "llvm/IR/DataLayout.h"
      44             : #include "llvm/IR/DebugInfoMetadata.h"
      45             : #include "llvm/IR/DebugLoc.h"
      46             : #include "llvm/IR/DerivedTypes.h"
      47             : #include "llvm/IR/Function.h"
      48             : #include "llvm/IR/GlobalValue.h"
      49             : #include "llvm/IR/Metadata.h"
      50             : #include "llvm/IR/Type.h"
      51             : #include "llvm/IR/Value.h"
      52             : #include "llvm/Support/Casting.h"
      53             : #include "llvm/Support/CodeGen.h"
      54             : #include "llvm/Support/Compiler.h"
      55             : #include "llvm/Support/Debug.h"
      56             : #include "llvm/Support/ErrorHandling.h"
      57             : #include "llvm/Support/KnownBits.h"
      58             : #include "llvm/Support/ManagedStatic.h"
      59             : #include "llvm/Support/MathExtras.h"
      60             : #include "llvm/Support/Mutex.h"
      61             : #include "llvm/Support/raw_ostream.h"
      62             : #include "llvm/Target/TargetLowering.h"
      63             : #include "llvm/Target/TargetMachine.h"
      64             : #include "llvm/Target/TargetOptions.h"
      65             : #include "llvm/Target/TargetRegisterInfo.h"
      66             : #include "llvm/Target/TargetSubtargetInfo.h"
      67             : #include <algorithm>
      68             : #include <cassert>
      69             : #include <cstdint>
      70             : #include <cstdlib>
      71             : #include <limits>
      72             : #include <set>
      73             : #include <string>
      74             : #include <utility>
      75             : #include <vector>
      76             : 
      77             : using namespace llvm;
      78             : 
      79             : /// makeVTList - Return an instance of the SDVTList struct initialized with the
      80             : /// specified members.
      81             : static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
      82    28761278 :   SDVTList Res = {VTs, NumVTs};
      83             :   return Res;
      84             : }
      85             : 
      86             : // Default null implementations of the callbacks.
      87           0 : void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
      88    13280601 : void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
      89             : 
      90             : #define DEBUG_TYPE "selectiondag"
      91             : 
      92             : static void NewSDValueDbgMsg(SDValue V, StringRef Msg) {
      93             :   DEBUG(
      94             :     dbgs() << Msg;
      95             :     V.dump();
      96             :   );
      97             : }
      98             : 
      99             : //===----------------------------------------------------------------------===//
     100             : //                              ConstantFPSDNode Class
     101             : //===----------------------------------------------------------------------===//
     102             : 
     103             : /// isExactlyValue - We don't rely on operator== working on double values, as
     104             : /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
     105             : /// As such, this method can be used to do an exact bit-for-bit comparison of
     106             : /// two floating point values.
     107       11414 : bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
     108       22828 :   return getValueAPF().bitwiseIsEqual(V);
     109             : }
     110             : 
     111        1500 : bool ConstantFPSDNode::isValueValidForType(EVT VT,
     112             :                                            const APFloat& Val) {
     113             :   assert(VT.isFloatingPoint() && "Can only convert between FP types");
     114             : 
     115             :   // convert modifies in place, so make a copy.
     116        3000 :   APFloat Val2 = APFloat(Val);
     117             :   bool losesInfo;
     118        1500 :   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
     119             :                       APFloat::rmNearestTiesToEven,
     120             :                       &losesInfo);
     121        3000 :   return !losesInfo;
     122             : }
     123             : 
     124             : //===----------------------------------------------------------------------===//
     125             : //                              ISD Namespace
     126             : //===----------------------------------------------------------------------===//
     127             : 
     128      470342 : bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
     129      266835 :   auto *BV = dyn_cast<BuildVectorSDNode>(N);
     130             :   if (!BV)
     131             :     return false;
     132             : 
     133      266835 :   APInt SplatUndef;
     134             :   unsigned SplatBitSize;
     135             :   bool HasUndefs;
     136      533670 :   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
     137      266835 :   return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
     138      533352 :                              EltSize) &&
     139      533352 :          EltSize == SplatBitSize;
     140             : }
     141             : 
     142             : // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
     143             : // specializations of the more general isConstantSplatVector()?
     144             : 
     145      278797 : bool ISD::isBuildVectorAllOnes(const SDNode *N) {
     146             :   // Look through a bit convert.
     147      723368 :   while (N->getOpcode() == ISD::BITCAST)
     148      110516 :     N = N->getOperand(0).getNode();
     149             : 
     150      278797 :   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
     151             : 
     152      313790 :   unsigned i = 0, e = N->getNumOperands();
     153             : 
     154             :   // Skip over all of the undef values.
     155      474831 :   while (i != e && N->getOperand(i).isUndef())
     156        1049 :     ++i;
     157             : 
     158             :   // Do not accept an all-undef vector.
     159      156895 :   if (i == e) return false;
     160             : 
     161             :   // Do not accept build_vectors that aren't all constants or which have non-~0
     162             :   // elements. We have to be a bit careful here, as the type of the constant
     163             :   // may not be the same as the type of the vector elements due to type
     164             :   // legalization (the elements are promoted to a legal type for the target and
     165             :   // a vector of a type may be legal when the base element type is not).
     166             :   // We only want to check enough bits to cover the vector elements, because
     167             :   // we care if the resultant vector is all ones, not whether the individual
     168             :   // constants are.
     169      313740 :   SDValue NotZero = N->getOperand(i);
     170      470610 :   unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
     171      145685 :   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
     172      291370 :     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
     173             :       return false;
     174        1670 :   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
     175        5010 :     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
     176             :       return false;
     177             :   } else
     178             :     return false;
     179             : 
     180             :   // Okay, we have at least one ~0 value, check to see if the rest match or are
     181             :   // undefs. Even with the above element type twiddling, this should be OK, as
     182             :   // the same type legalization should have applied to all the elements.
     183      478209 :   for (++i; i != e; ++i)
     184      745503 :     if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
     185             :       return false;
     186             :   return true;
     187             : }
     188             : 
     189     1241855 : bool ISD::isBuildVectorAllZeros(const SDNode *N) {
     190             :   // Look through a bit convert.
     191     3265996 :   while (N->getOpcode() == ISD::BITCAST)
     192      521524 :     N = N->getOperand(0).getNode();
     193             : 
     194     1241855 :   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
     195             : 
     196      641388 :   bool IsAllUndef = true;
     197     3286972 :   for (const SDValue &Op : N->op_values()) {
     198     1293238 :     if (Op.isUndef())
     199        5273 :       continue;
     200     1287965 :     IsAllUndef = false;
     201             :     // Do not accept build_vectors that aren't all constants or which have non-0
     202             :     // elements. We have to be a bit careful here, as the type of the constant
     203             :     // may not be the same as the type of the vector elements due to type
     204             :     // legalization (the elements are promoted to a legal type for the target
     205             :     // and a vector of a type may be legal when the base element type is not).
     206             :     // We only want to check enough bits to cover the vector elements, because
     207             :     // we care if the resultant vector is all zeros, not whether the individual
     208             :     // constants are.
     209     3863895 :     unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
     210     1257478 :     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
     211     1257478 :       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
     212             :         return false;
     213       17647 :     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
     214       35294 :       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
     215             :         return false;
     216             :     } else
     217             :       return false;
     218             :   }
     219             : 
     220             :   // Do not accept an all-undef vector.
     221       59108 :   if (IsAllUndef)
     222             :     return false;
     223       59083 :   return true;
     224             : }
     225             : 
     226    11367550 : bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
     227    11367550 :   if (N->getOpcode() != ISD::BUILD_VECTOR)
     228             :     return false;
     229             : 
     230      949417 :   for (const SDValue &Op : N->op_values()) {
     231      433592 :     if (Op.isUndef())
     232        2802 :       continue;
     233             :     if (!isa<ConstantSDNode>(Op))
     234             :       return false;
     235             :   }
     236             :   return true;
     237             : }
     238             : 
     239      923243 : bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
     240      923243 :   if (N->getOpcode() != ISD::BUILD_VECTOR)
     241             :     return false;
     242             : 
     243       50859 :   for (const SDValue &Op : N->op_values()) {
     244       20532 :     if (Op.isUndef())
     245         655 :       continue;
     246             :     if (!isa<ConstantFPSDNode>(Op))
     247             :       return false;
     248             :   }
     249             :   return true;
     250             : }
     251             : 
     252      319042 : bool ISD::allOperandsUndef(const SDNode *N) {
     253             :   // Return false if the node has no operands.
     254             :   // This is "logically inconsistent" with the definition of "all" but
     255             :   // is probably the desired behavior.
     256      638084 :   if (N->getNumOperands() == 0)
     257             :     return false;
     258             : 
     259      964773 :   for (const SDValue &Op : N->op_values())
     260      322829 :     if (!Op.isUndef())
     261             :       return false;
     262             : 
     263             :   return true;
     264             : }
     265             : 
     266         308 : ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
     267         308 :   switch (ExtType) {
     268           0 :   case ISD::EXTLOAD:
     269           0 :     return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
     270             :   case ISD::SEXTLOAD:
     271             :     return ISD::SIGN_EXTEND;
     272         195 :   case ISD::ZEXTLOAD:
     273         195 :     return ISD::ZERO_EXTEND;
     274             :   default:
     275             :     break;
     276             :   }
     277             : 
     278           0 :   llvm_unreachable("Invalid LoadExtType");
     279             : }
     280             : 
     281      341050 : ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
     282             :   // To perform this operation, we just need to swap the L and G bits of the
     283             :   // operation.
     284      341050 :   unsigned OldL = (Operation >> 2) & 1;
     285      341050 :   unsigned OldG = (Operation >> 1) & 1;
     286      682100 :   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
     287      341050 :                        (OldL << 1) |       // New G bit
     288      341050 :                        (OldG << 2));       // New L bit.
     289             : }
     290             : 
     291       56755 : ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
     292       56755 :   unsigned Operation = Op;
     293       56755 :   if (isInteger)
     294       55356 :     Operation ^= 7;   // Flip L, G, E bits, but not U.
     295             :   else
     296        1399 :     Operation ^= 15;  // Flip all of the condition bits.
     297             : 
     298       56755 :   if (Operation > ISD::SETTRUE2)
     299         295 :     Operation &= ~8;  // Don't let N and U bits get set.
     300             : 
     301       56755 :   return ISD::CondCode(Operation);
     302             : }
     303             : 
     304             : /// For an integer comparison, return 1 if the comparison is a signed operation
     305             : /// and 2 if the result is an unsigned comparison. Return zero if the operation
     306             : /// does not depend on the sign of the input (setne and seteq).
     307             : static int isSignedOp(ISD::CondCode Opcode) {
     308             :   switch (Opcode) {
     309           0 :   default: llvm_unreachable("Illegal integer setcc operation!");
     310             :   case ISD::SETEQ:
     311             :   case ISD::SETNE: return 0;
     312             :   case ISD::SETLT:
     313             :   case ISD::SETLE:
     314             :   case ISD::SETGT:
     315             :   case ISD::SETGE: return 1;
     316             :   case ISD::SETULT:
     317             :   case ISD::SETULE:
     318             :   case ISD::SETUGT:
     319             :   case ISD::SETUGE: return 2;
     320             :   }
     321             : }
     322             : 
     323          37 : ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
     324             :                                        bool IsInteger) {
     325          75 :   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
     326             :     // Cannot fold a signed integer setcc with an unsigned integer setcc.
     327             :     return ISD::SETCC_INVALID;
     328             : 
     329          32 :   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
     330             : 
     331             :   // If the N and U bits get set, then the resultant comparison DOES suddenly
     332             :   // care about orderedness, and it is true when ordered.
     333          32 :   if (Op > ISD::SETTRUE2)
     334          17 :     Op &= ~16;     // Clear the U bit if the N bit is set.
     335             : 
     336             :   // Canonicalize illegal integer setcc's.
     337          32 :   if (IsInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
     338           0 :     Op = ISD::SETNE;
     339             : 
     340             :   return ISD::CondCode(Op);
     341             : }
     342             : 
     343          33 : ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
     344             :                                         bool IsInteger) {
     345          35 :   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
     346             :     // Cannot fold a signed setcc with an unsigned setcc.
     347             :     return ISD::SETCC_INVALID;
     348             : 
     349             :   // Combine all of the condition bits.
     350          33 :   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
     351             : 
     352             :   // Canonicalize illegal integer setcc's.
     353          33 :   if (IsInteger) {
     354           1 :     switch (Result) {
     355             :     default: break;
     356           0 :     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
     357           1 :     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
     358           1 :     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
     359           0 :     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
     360           0 :     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
     361             :     }
     362             :   }
     363             : 
     364             :   return Result;
     365             : }
     366             : 
     367             : //===----------------------------------------------------------------------===//
     368             : //                           SDNode Profile Support
     369             : //===----------------------------------------------------------------------===//
     370             : 
     371             : /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
     372             : static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
     373    64841916 :   ID.AddInteger(OpC);
     374             : }
     375             : 
     376             : /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
     377             : /// solely with their pointer.
     378             : static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
     379    64841916 :   ID.AddPointer(VTList.VTs);
     380             : }
     381             : 
     382             : /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
     383    31710809 : static void AddNodeIDOperands(FoldingSetNodeID &ID,
     384             :                               ArrayRef<SDValue> Ops) {
     385   103278986 :   for (auto& Op : Ops) {
     386    39857368 :     ID.AddPointer(Op.getNode());
     387    39857368 :     ID.AddInteger(Op.getResNo());
     388             :   }
     389    31710809 : }
     390             : 
     391             : /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
     392    33131107 : static void AddNodeIDOperands(FoldingSetNodeID &ID,
     393             :                               ArrayRef<SDUse> Ops) {
     394   118517359 :   for (auto& Op : Ops) {
     395    52255145 :     ID.AddPointer(Op.getNode());
     396    52255145 :     ID.AddInteger(Op.getResNo());
     397             :   }
     398    33131107 : }
     399             : 
     400    31710809 : static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
     401             :                           SDVTList VTList, ArrayRef<SDValue> OpList) {
     402    63421618 :   AddNodeIDOpcode(ID, OpC);
     403    63421618 :   AddNodeIDValueTypes(ID, VTList);
     404    31710809 :   AddNodeIDOperands(ID, OpList);
     405    31710809 : }
     406             : 
     407             : /// If this is an SDNode with special info, add this info to the NodeID data.
     408    33277525 : static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
     409    66555050 :   switch (N->getOpcode()) {
     410           0 :   case ISD::TargetExternalSymbol:
     411             :   case ISD::ExternalSymbol:
     412             :   case ISD::MCSymbol:
     413           0 :     llvm_unreachable("Should only be used on nodes with operands");
     414             :   default: break;  // Normal nodes don't need extra info.
     415     8055137 :   case ISD::TargetConstant:
     416             :   case ISD::Constant: {
     417     8055137 :     const ConstantSDNode *C = cast<ConstantSDNode>(N);
     418     8055137 :     ID.AddPointer(C->getConstantIntValue());
     419     8055137 :     ID.AddBoolean(C->isOpaque());
     420             :     break;
     421             :   }
     422       23358 :   case ISD::TargetConstantFP:
     423             :   case ISD::ConstantFP:
     424       23358 :     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
     425       23358 :     break;
     426     1420015 :   case ISD::TargetGlobalAddress:
     427             :   case ISD::GlobalAddress:
     428             :   case ISD::TargetGlobalTLSAddress:
     429             :   case ISD::GlobalTLSAddress: {
     430     1420015 :     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
     431     1420015 :     ID.AddPointer(GA->getGlobal());
     432     1420015 :     ID.AddInteger(GA->getOffset());
     433     1420015 :     ID.AddInteger(GA->getTargetFlags());
     434     1420015 :     break;
     435             :   }
     436      119757 :   case ISD::BasicBlock:
     437      119757 :     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
     438      119757 :     break;
     439     3816861 :   case ISD::Register:
     440     3816861 :     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
     441     3816861 :     break;
     442      157312 :   case ISD::RegisterMask:
     443      157312 :     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
     444      157312 :     break;
     445         458 :   case ISD::SRCVALUE:
     446         458 :     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
     447         458 :     break;
     448      313509 :   case ISD::FrameIndex:
     449             :   case ISD::TargetFrameIndex:
     450      313509 :     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
     451      313509 :     break;
     452         186 :   case ISD::JumpTable:
     453             :   case ISD::TargetJumpTable:
     454         186 :     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
     455         186 :     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
     456         186 :     break;
     457       20587 :   case ISD::ConstantPool:
     458             :   case ISD::TargetConstantPool: {
     459       20587 :     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
     460       20587 :     ID.AddInteger(CP->getAlignment());
     461       20587 :     ID.AddInteger(CP->getOffset());
     462       20587 :     if (CP->isMachineConstantPoolEntry())
     463         102 :       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
     464             :     else
     465       20485 :       ID.AddPointer(CP->getConstVal());
     466       20587 :     ID.AddInteger(CP->getTargetFlags());
     467       20587 :     break;
     468             :   }
     469           0 :   case ISD::TargetIndex: {
     470           0 :     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
     471           0 :     ID.AddInteger(TI->getIndex());
     472           0 :     ID.AddInteger(TI->getOffset());
     473           0 :     ID.AddInteger(TI->getTargetFlags());
     474           0 :     break;
     475             :   }
     476     1203465 :   case ISD::LOAD: {
     477     1203465 :     const LoadSDNode *LD = cast<LoadSDNode>(N);
     478     2406930 :     ID.AddInteger(LD->getMemoryVT().getRawBits());
     479     2406930 :     ID.AddInteger(LD->getRawSubclassData());
     480     2406930 :     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
     481     1203465 :     break;
     482             :   }
     483     2466151 :   case ISD::STORE: {
     484     2466151 :     const StoreSDNode *ST = cast<StoreSDNode>(N);
     485     4932302 :     ID.AddInteger(ST->getMemoryVT().getRawBits());
     486     4932302 :     ID.AddInteger(ST->getRawSubclassData());
     487     4932302 :     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
     488     2466151 :     break;
     489             :   }
     490        8912 :   case ISD::ATOMIC_CMP_SWAP:
     491             :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     492             :   case ISD::ATOMIC_SWAP:
     493             :   case ISD::ATOMIC_LOAD_ADD:
     494             :   case ISD::ATOMIC_LOAD_SUB:
     495             :   case ISD::ATOMIC_LOAD_AND:
     496             :   case ISD::ATOMIC_LOAD_OR:
     497             :   case ISD::ATOMIC_LOAD_XOR:
     498             :   case ISD::ATOMIC_LOAD_NAND:
     499             :   case ISD::ATOMIC_LOAD_MIN:
     500             :   case ISD::ATOMIC_LOAD_MAX:
     501             :   case ISD::ATOMIC_LOAD_UMIN:
     502             :   case ISD::ATOMIC_LOAD_UMAX:
     503             :   case ISD::ATOMIC_LOAD:
     504             :   case ISD::ATOMIC_STORE: {
     505        8912 :     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
     506       17824 :     ID.AddInteger(AT->getMemoryVT().getRawBits());
     507       17824 :     ID.AddInteger(AT->getRawSubclassData());
     508       17824 :     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
     509        8912 :     break;
     510             :   }
     511         116 :   case ISD::PREFETCH: {
     512         116 :     const MemSDNode *PF = cast<MemSDNode>(N);
     513         116 :     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
     514         116 :     break;
     515             :   }
     516       39011 :   case ISD::VECTOR_SHUFFLE: {
     517       39011 :     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
     518      627890 :     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
     519      588879 :          i != e; ++i)
     520     1099736 :       ID.AddInteger(SVN->getMaskElt(i));
     521             :     break;
     522             :   }
     523          58 :   case ISD::TargetBlockAddress:
     524             :   case ISD::BlockAddress: {
     525          58 :     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
     526          58 :     ID.AddPointer(BA->getBlockAddress());
     527          58 :     ID.AddInteger(BA->getOffset());
     528          58 :     ID.AddInteger(BA->getTargetFlags());
     529          58 :     break;
     530             :   }
     531             :   } // end switch (N->getOpcode())
     532             : 
     533             :   // Target specific memory nodes could also have address spaces to check.
     534    33277525 :   if (N->isTargetMemoryOpcode())
     535       15240 :     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
     536    33277525 : }
     537             : 
     538             : /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
     539             : /// data.
     540    33131107 : static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
     541    99393321 :   AddNodeIDOpcode(ID, N->getOpcode());
     542             :   // Add the return value info.
     543    99393321 :   AddNodeIDValueTypes(ID, N->getVTList());
     544             :   // Add the operand info.
     545    33131107 :   AddNodeIDOperands(ID, N->ops());
     546             : 
     547             :   // Handle SDNode leafs with special info.
     548    33131107 :   AddNodeIDCustom(ID, N);
     549    33131107 : }
     550             : 
     551             : //===----------------------------------------------------------------------===//
     552             : //                              SelectionDAG Class
     553             : //===----------------------------------------------------------------------===//
     554             : 
     555             : /// doNotCSE - Return true if CSE should not be performed for this node.
     556     6272793 : static bool doNotCSE(SDNode *N) {
     557    18818379 :   if (N->getValueType(0) == MVT::Glue)
     558             :     return true; // Never CSE anything that produces a flag.
     559             : 
     560    12544584 :   switch (N->getOpcode()) {
     561             :   default: break;
     562             :   case ISD::HANDLENODE:
     563             :   case ISD::EH_LABEL:
     564             :     return true;   // Never CSE these nodes.
     565             :   }
     566             : 
     567             :   // Check that remaining values produced are not flags.
     568    13432432 :   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
     569     5338761 :     if (N->getValueType(i) == MVT::Glue)
     570             :       return true; // Never CSE anything that produces a flag.
     571             : 
     572             :   return false;
     573             : }
     574             : 
     575             : /// RemoveDeadNodes - This method deletes all unreachable nodes in the
     576             : /// SelectionDAG.
     577     1377363 : void SelectionDAG::RemoveDeadNodes() {
     578             :   // Create a dummy node (which is not added to allnodes), that adds a reference
     579             :   // to the root node, preventing it from being deleted.
     580     2754726 :   HandleSDNode Dummy(getRoot());
     581             : 
     582     2754726 :   SmallVector<SDNode*, 128> DeadNodes;
     583             : 
     584             :   // Add all obviously-dead nodes to the DeadNodes worklist.
     585    45393403 :   for (SDNode &Node : allnodes())
     586    44016040 :     if (Node.use_empty())
     587      667516 :       DeadNodes.push_back(&Node);
     588             : 
     589     1377363 :   RemoveDeadNodes(DeadNodes);
     590             : 
     591             :   // If the root changed (e.g. it was a dead load, update the root).
     592     1377363 :   setRoot(Dummy.getValue());
     593     1377363 : }
     594             : 
     595             : /// RemoveDeadNodes - This method deletes the unreachable nodes in the
     596             : /// given list, and any nodes that become unreachable as a result.
     597     4022039 : void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
     598             : 
     599             :   // Process the worklist, deleting the nodes and adding their uses to the
     600             :   // worklist.
     601     8363793 :   while (!DeadNodes.empty()) {
     602     4341754 :     SDNode *N = DeadNodes.pop_back_val();
     603             :     // Skip to next node if we've already managed to delete the node. This could
     604             :     // happen if replacing a node causes a node previously added to the node to
     605             :     // be deleted.
     606     4341754 :     if (N->getOpcode() == ISD::DELETED_NODE)
     607           0 :       continue;
     608             : 
     609     9434279 :     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
     610     5092525 :       DUL->NodeDeleted(N, nullptr);
     611             : 
     612             :     // Take the node out of the appropriate CSE map.
     613     4341754 :     RemoveNodeFromCSEMaps(N);
     614             : 
     615             :     // Next, brutally remove the operand list.  This is safe to do, as there are
     616             :     // no cycles in the graph.
     617    14886054 :     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
     618     6202546 :       SDUse &Use = *I++;
     619     6202546 :       SDNode *Operand = Use.getNode();
     620     6202546 :       Use.set(SDValue());
     621             : 
     622             :       // Now that we removed this operand, see if there are no uses of it left.
     623     6202546 :       if (Operand->use_empty())
     624     2094771 :         DeadNodes.push_back(Operand);
     625             :     }
     626             : 
     627     4341754 :     DeallocateNode(N);
     628             :   }
     629     4022039 : }
     630             : 
     631      345766 : void SelectionDAG::RemoveDeadNode(SDNode *N){
     632      691532 :   SmallVector<SDNode*, 16> DeadNodes(1, N);
     633             : 
     634             :   // Create a dummy node that adds a reference to the root node, preventing
     635             :   // it from being deleted.  (This matters if the root is an operand of the
     636             :   // dead node.)
     637      691532 :   HandleSDNode Dummy(getRoot());
     638             : 
     639      345766 :   RemoveDeadNodes(DeadNodes);
     640      345766 : }
     641             : 
     642     5546231 : void SelectionDAG::DeleteNode(SDNode *N) {
     643             :   // First take this out of the appropriate CSE map.
     644     5546231 :   RemoveNodeFromCSEMaps(N);
     645             : 
     646             :   // Finally, remove uses due to operands of this node, remove from the
     647             :   // AllNodes list, and delete the node.
     648     5546231 :   DeleteNodeNotInCSEMaps(N);
     649     5546231 : }
     650             : 
     651     5561513 : void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
     652             :   assert(N->getIterator() != AllNodes.begin() &&
     653             :          "Cannot delete the entry node!");
     654             :   assert(N->use_empty() && "Cannot delete a node that is not dead!");
     655             : 
     656             :   // Drop all of the operands and decrement used node's use counts.
     657     5561513 :   N->DropOperands();
     658             : 
     659     5561513 :   DeallocateNode(N);
     660     5561513 : }
     661             : 
     662    17850204 : void SDDbgInfo::erase(const SDNode *Node) {
     663    17850204 :   DbgValMapType::iterator I = DbgValMap.find(Node);
     664    53550612 :   if (I == DbgValMap.end())
     665    17842715 :     return;
     666       38272 :   for (auto &Val: I->second)
     667       15805 :     Val->setIsInvalidated();
     668       14978 :   DbgValMap.erase(I);
     669             : }
     670             : 
     671    17850204 : void SelectionDAG::DeallocateNode(SDNode *N) {
     672             :   // If we have operands, deallocate them.
     673    17850204 :   removeOperands(N);
     674             : 
     675    53550612 :   NodeAllocator.Deallocate(AllNodes.remove(N));
     676             : 
     677             :   // Set the opcode to DELETED_NODE to help catch bugs when node
     678             :   // memory is reallocated.
     679             :   // FIXME: There are places in SDag that have grown a dependency on the opcode
     680             :   // value in the released node.
     681             :   __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
     682    17850204 :   N->NodeType = ISD::DELETED_NODE;
     683             : 
     684             :   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
     685             :   // them and forget about that node.
     686    17850204 :   DbgInfo->erase(N);
     687    17850204 : }
     688             : 
     689             : #ifndef NDEBUG
     690             : /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
     691             : static void VerifySDNode(SDNode *N) {
     692             :   switch (N->getOpcode()) {
     693             :   default:
     694             :     break;
     695             :   case ISD::BUILD_PAIR: {
     696             :     EVT VT = N->getValueType(0);
     697             :     assert(N->getNumValues() == 1 && "Too many results!");
     698             :     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
     699             :            "Wrong return type!");
     700             :     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
     701             :     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
     702             :            "Mismatched operand types!");
     703             :     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
     704             :            "Wrong operand type!");
     705             :     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
     706             :            "Wrong return type size");
     707             :     break;
     708             :   }
     709             :   case ISD::BUILD_VECTOR: {
     710             :     assert(N->getNumValues() == 1 && "Too many results!");
     711             :     assert(N->getValueType(0).isVector() && "Wrong return type!");
     712             :     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
     713             :            "Wrong number of operands!");
     714             :     EVT EltVT = N->getValueType(0).getVectorElementType();
     715             :     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
     716             :       assert((I->getValueType() == EltVT ||
     717             :              (EltVT.isInteger() && I->getValueType().isInteger() &&
     718             :               EltVT.bitsLE(I->getValueType()))) &&
     719             :             "Wrong operand type!");
     720             :       assert(I->getValueType() == N->getOperand(0).getValueType() &&
     721             :              "Operands must all have the same type");
     722             :     }
     723             :     break;
     724             :   }
     725             :   }
     726             : }
     727             : #endif // NDEBUG
     728             : 
     729             : /// \brief Insert a newly allocated node into the DAG.
     730             : ///
     731             : /// Handles insertion into the all nodes list and CSE map, as well as
     732             : /// verification and other common operations when a new node is allocated.
     733    18149641 : void SelectionDAG::InsertNode(SDNode *N) {
     734    36299282 :   AllNodes.push_back(N);
     735             : #ifndef NDEBUG
     736             :   N->PersistentId = NextPersistentId++;
     737             :   VerifySDNode(N);
     738             : #endif
     739    18149641 : }
     740             : 
     741             : /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
     742             : /// correspond to it.  This is useful when we're about to delete or repurpose
     743             : /// the node.  We don't want future request for structurally identical nodes
     744             : /// to return N anymore.
     745    18864180 : bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
     746    18864180 :   bool Erased = false;
     747    37728360 :   switch (N->getOpcode()) {
     748             :   case ISD::HANDLENODE: return false;  // noop.
     749       96670 :   case ISD::CONDCODE:
     750             :     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
     751             :            "Cond code doesn't exist!");
     752      193340 :     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
     753       96670 :     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
     754       96670 :     break;
     755        5306 :   case ISD::ExternalSymbol:
     756       15918 :     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
     757        5306 :     break;
     758         246 :   case ISD::TargetExternalSymbol: {
     759         246 :     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
     760         492 :     Erased = TargetExternalSymbols.erase(
     761         738 :                std::pair<std::string,unsigned char>(ESN->getSymbol(),
     762             :                                                     ESN->getTargetFlags()));
     763         246 :     break;
     764             :   }
     765           0 :   case ISD::MCSymbol: {
     766           0 :     auto *MCSN = cast<MCSymbolSDNode>(N);
     767           0 :     Erased = MCSymbols.erase(MCSN->getMCSymbol());
     768           0 :     break;
     769             :   }
     770       40970 :   case ISD::VALUETYPE: {
     771       40970 :     EVT VT = cast<VTSDNode>(N)->getVT();
     772       40970 :     if (VT.isExtended()) {
     773        3136 :       Erased = ExtendedValueTypeNodes.erase(VT);
     774             :     } else {
     775       78804 :       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
     776       39402 :       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
     777             :     }
     778             :     break;
     779             :   }
     780    18689867 :   default:
     781             :     // Remove it from the CSE Map.
     782             :     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
     783             :     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
     784    37379734 :     Erased = CSEMap.RemoveNode(N);
     785    18689867 :     break;
     786             :   }
     787             : #ifndef NDEBUG
     788             :   // Verify that the node was actually in one of the CSE maps, unless it has a
     789             :   // flag result (which cannot be CSE'd) or is one of the special cases that are
     790             :   // not subject to CSE.
     791             :   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
     792             :       !N->isMachineOpcode() && !doNotCSE(N)) {
     793             :     N->dump(this);
     794             :     dbgs() << "\n";
     795             :     llvm_unreachable("Node is not in map!");
     796             :   }
     797             : #endif
     798             :   return Erased;
     799             : }
     800             : 
     801             : /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
     802             : /// maps and modified in place. Add it back to the CSE maps, unless an identical
     803             : /// node already exists, in which case transfer all its users to the existing
     804             : /// node. This transfer can potentially trigger recursive merging.
     805             : void
     806     6116165 : SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
     807             :   // For node types that aren't CSE'd, just act as if no identical node
     808             :   // already exists.
     809     6116165 :   if (!doNotCSE(N)) {
     810    10615694 :     SDNode *Existing = CSEMap.GetOrInsertNode(N);
     811     5307847 :     if (Existing != N) {
     812             :       // If there was already an existing matching node, use ReplaceAllUsesWith
     813             :       // to replace the dead one with the existing one.  This can cause
     814             :       // recursive merging of other unrelated nodes down the line.
     815       15282 :       ReplaceAllUsesWith(N, Existing);
     816             : 
     817             :       // N is now dead. Inform the listeners and delete it.
     818       65122 :       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
     819       49840 :         DUL->NodeDeleted(N, Existing);
     820       15282 :       DeleteNodeNotInCSEMaps(N);
     821       15282 :       return;
     822             :     }
     823             :   }
     824             : 
     825             :   // If the node doesn't already exist, we updated it.  Inform listeners.
     826    19870769 :   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
     827    13769886 :     DUL->NodeUpdated(N);
     828             : }
     829             : 
     830             : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
     831             : /// were replaced with those specified.  If this node is never memoized,
     832             : /// return null, otherwise return a pointer to the slot it would take.  If a
     833             : /// node already exists with these operands, the slot will be non-null.
     834        1666 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
     835             :                                            void *&InsertPos) {
     836        1666 :   if (doNotCSE(N))
     837             :     return nullptr;
     838             : 
     839        1666 :   SDValue Ops[] = { Op };
     840        1666 :   FoldingSetNodeID ID;
     841        4998 :   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
     842        1666 :   AddNodeIDCustom(ID, N);
     843        3332 :   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
     844        1666 :   if (Node)
     845           0 :     Node->intersectFlagsWith(N->getFlags());
     846        1666 :   return Node;
     847             : }
     848             : 
     849             : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
     850             : /// were replaced with those specified.  If this node is never memoized,
     851             : /// return null, otherwise return a pointer to the slot it would take.  If a
     852             : /// node already exists with these operands, the slot will be non-null.
     853       53131 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
     854             :                                            SDValue Op1, SDValue Op2,
     855             :                                            void *&InsertPos) {
     856       53131 :   if (doNotCSE(N))
     857             :     return nullptr;
     858             : 
     859       53131 :   SDValue Ops[] = { Op1, Op2 };
     860       53131 :   FoldingSetNodeID ID;
     861      159393 :   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
     862       53131 :   AddNodeIDCustom(ID, N);
     863      106262 :   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
     864       53131 :   if (Node)
     865        1628 :     Node->intersectFlagsWith(N->getFlags());
     866       53131 :   return Node;
     867             : }
     868             : 
     869             : /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
     870             : /// were replaced with those specified.  If this node is never memoized,
     871             : /// return null, otherwise return a pointer to the slot it would take.  If a
     872             : /// node already exists with these operands, the slot will be non-null.
     873      101831 : SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
     874             :                                            void *&InsertPos) {
     875      101831 :   if (doNotCSE(N))
     876             :     return nullptr;
     877             : 
     878       91621 :   FoldingSetNodeID ID;
     879      274863 :   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
     880       91621 :   AddNodeIDCustom(ID, N);
     881      183242 :   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
     882       91621 :   if (Node)
     883        1058 :     Node->intersectFlagsWith(N->getFlags());
     884       91621 :   return Node;
     885             : }
     886             : 
     887      289851 : unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
     888      579702 :   Type *Ty = VT == MVT::iPTR ?
     889           0 :                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
     890      579702 :                    VT.getTypeForEVT(*getContext());
     891             : 
     892      579702 :   return getDataLayout().getABITypeAlignment(Ty);
     893             : }
     894             : 
     895             : // EntryNode could meaningfully have debug info if we can find it...
     896       18528 : SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
     897             :     : TM(tm), OptLevel(OL),
     898       74112 :       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
     899      333504 :       Root(getEntryNode()) {
     900       18528 :   InsertNode(&EntryNode);
     901       18528 :   DbgInfo = new SDDbgInfo();
     902       18528 : }
     903             : 
     904      142331 : void SelectionDAG::init(MachineFunction &NewMF,
     905             :                         OptimizationRemarkEmitter &NewORE,
     906             :                         Pass *PassPtr) {
     907      142331 :   MF = &NewMF;
     908      142331 :   SDAGISelPass = PassPtr;
     909      142331 :   ORE = &NewORE;
     910      284662 :   TLI = getSubtarget().getTargetLowering();
     911      284662 :   TSI = getSubtarget().getSelectionDAGInfo();
     912      142331 :   Context = &MF->getFunction()->getContext();
     913      142331 : }
     914             : 
     915      239459 : SelectionDAG::~SelectionDAG() {
     916             :   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
     917       18419 :   allnodes_clear();
     918       36840 :   OperandRecycler.clear(OperandAllocator);
     919       18420 :   delete DbgInfo;
     920       18420 : }
     921             : 
     922      298428 : void SelectionDAG::allnodes_clear() {
     923             :   assert(&*AllNodes.begin() == &EntryNode);
     924      895284 :   AllNodes.remove(AllNodes.begin());
     925    24437667 :   while (!AllNodes.empty())
     926    15893874 :     DeallocateNode(&AllNodes.front());
     927             : #ifndef NDEBUG
     928             :   NextPersistentId = 0;
     929             : #endif
     930      298428 : }
     931             : 
     932     5114279 : SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
     933             :                                           void *&InsertPos) {
     934    10228558 :   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
     935     5114279 :   if (N) {
     936     3131247 :     switch (N->getOpcode()) {
     937             :     default: break;
     938           0 :     case ISD::Constant:
     939             :     case ISD::ConstantFP:
     940           0 :       llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
     941             :                        "debug location.  Use another overload.");
     942             :     }
     943             :   }
     944     5114279 :   return N;
     945             : }
     946             : 
     947    26596625 : SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
     948             :                                           const SDLoc &DL, void *&InsertPos) {
     949    53193250 :   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
     950    26596625 :   if (N) {
     951     9302157 :     switch (N->getOpcode()) {
     952     2336620 :     case ISD::Constant:
     953             :     case ISD::ConstantFP:
     954             :       // Erase debug location from the node if the node is used at several
     955             :       // different places. Do not propagate one location to all uses as it
     956             :       // will cause a worse single stepping debugging experience.
     957     4673240 :       if (N->getDebugLoc() != DL.getDebugLoc())
     958     2313500 :         N->setDebugLoc(DebugLoc());
     959             :       break;
     960     6965537 :     default:
     961             :       // When the node's point of use is located earlier in the instruction
     962             :       // sequence than its prior point of use, update its debug info to the
     963             :       // earlier location.
     964    13931074 :       if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
     965      312548 :         N->setDebugLoc(DL.getDebugLoc());
     966             :       break;
     967             :     }
     968             :   }
     969    26596625 :   return N;
     970             : }
     971             : 
     972      280008 : void SelectionDAG::clear() {
     973      280008 :   allnodes_clear();
     974      560016 :   OperandRecycler.clear(OperandAllocator);
     975      280008 :   OperandAllocator.Reset();
     976      280008 :   CSEMap.clear();
     977             : 
     978      560016 :   ExtendedValueTypeNodes.clear();
     979      280008 :   ExternalSymbols.clear();
     980      560016 :   TargetExternalSymbols.clear();
     981      280008 :   MCSymbols.clear();
     982     1120032 :   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
     983             :             static_cast<CondCodeSDNode*>(nullptr));
     984     1120032 :   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
     985             :             static_cast<SDNode*>(nullptr));
     986             : 
     987      280008 :   EntryNode.UseList = nullptr;
     988      280008 :   InsertNode(&EntryNode);
     989      280008 :   Root = getEntryNode();
     990      560016 :   DbgInfo->clear();
     991      280008 : }
     992             : 
     993           6 : SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
     994          12 :   return VT.bitsGT(Op.getValueType())
     995           6 :              ? getNode(ISD::FP_EXTEND, DL, VT, Op)
     996          18 :              : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
     997             : }
     998             : 
     999       43585 : SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
    1000       87170 :   return VT.bitsGT(Op.getValueType()) ?
    1001       47316 :     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
    1002      130755 :     getNode(ISD::TRUNCATE, DL, VT, Op);
    1003             : }
    1004             : 
    1005      128712 : SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
    1006      257424 :   return VT.bitsGT(Op.getValueType()) ?
    1007      146813 :     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
    1008      386136 :     getNode(ISD::TRUNCATE, DL, VT, Op);
    1009             : }
    1010             : 
    1011      169410 : SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
    1012      338820 :   return VT.bitsGT(Op.getValueType()) ?
    1013      176316 :     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
    1014      508230 :     getNode(ISD::TRUNCATE, DL, VT, Op);
    1015             : }
    1016             : 
    1017         448 : SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
    1018             :                                         EVT OpVT) {
    1019         896 :   if (VT.bitsLE(Op.getValueType()))
    1020         230 :     return getNode(ISD::TRUNCATE, SL, VT, Op);
    1021             : 
    1022         218 :   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
    1023         436 :   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
    1024             : }
    1025             : 
    1026       94805 : SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
    1027             :   assert(!VT.isVector() &&
    1028             :          "getZeroExtendInReg should use the vector element type instead of "
    1029             :          "the vector type!");
    1030      189731 :   if (Op.getValueType() == VT) return Op;
    1031       94784 :   unsigned BitWidth = Op.getScalarValueSizeInBits();
    1032             :   APInt Imm = APInt::getLowBitsSet(BitWidth,
    1033       94784 :                                    VT.getSizeInBits());
    1034             :   return getNode(ISD::AND, DL, Op.getValueType(), Op,
    1035      284352 :                  getConstant(Imm, DL, Op.getValueType()));
    1036             : }
    1037             : 
    1038         254 : SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL,
    1039             :                                               EVT VT) {
    1040             :   assert(VT.isVector() && "This DAG node is restricted to vector types.");
    1041             :   assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
    1042             :          "The sizes of the input and result must match in order to perform the "
    1043             :          "extend in-register.");
    1044             :   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
    1045             :          "The destination vector type must have fewer lanes than the input.");
    1046         254 :   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
    1047             : }
    1048             : 
    1049        1682 : SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL,
    1050             :                                                EVT VT) {
    1051             :   assert(VT.isVector() && "This DAG node is restricted to vector types.");
    1052             :   assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
    1053             :          "The sizes of the input and result must match in order to perform the "
    1054             :          "extend in-register.");
    1055             :   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
    1056             :          "The destination vector type must have fewer lanes than the input.");
    1057        1682 :   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
    1058             : }
    1059             : 
    1060        1426 : SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL,
    1061             :                                                EVT VT) {
    1062             :   assert(VT.isVector() && "This DAG node is restricted to vector types.");
    1063             :   assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
    1064             :          "The sizes of the input and result must match in order to perform the "
    1065             :          "extend in-register.");
    1066             :   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
    1067             :          "The destination vector type must have fewer lanes than the input.");
    1068        1426 :   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
    1069             : }
    1070             : 
    1071             : /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
    1072       24866 : SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
    1073       24866 :   EVT EltVT = VT.getScalarType();
    1074             :   SDValue NegOne =
    1075       49732 :     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
    1076       24866 :   return getNode(ISD::XOR, DL, VT, Val, NegOne);
    1077             : }
    1078             : 
    1079          16 : SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
    1080          16 :   EVT EltVT = VT.getScalarType();
    1081          16 :   SDValue TrueValue;
    1082          16 :   switch (TLI->getBooleanContents(VT)) {
    1083           0 :     case TargetLowering::ZeroOrOneBooleanContent:
    1084             :     case TargetLowering::UndefinedBooleanContent:
    1085           0 :       TrueValue = getConstant(1, DL, VT);
    1086           0 :       break;
    1087          16 :     case TargetLowering::ZeroOrNegativeOneBooleanContent:
    1088          32 :       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL,
    1089             :                               VT);
    1090          16 :       break;
    1091             :   }
    1092          16 :   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
    1093             : }
    1094             : 
    1095     6698949 : SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
    1096             :                                   bool isT, bool isO) {
    1097     6698949 :   EVT EltVT = VT.getScalarType();
    1098             :   assert((EltVT.getSizeInBits() >= 64 ||
    1099             :          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
    1100             :          "getConstant with a uint64_t value that doesn't fit in the type!");
    1101    20096847 :   return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
    1102             : }
    1103             : 
    1104     8326879 : SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
    1105             :                                   bool isT, bool isO) {
    1106     8326879 :   return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
    1107             : }
    1108             : 
    1109     8999477 : SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
    1110             :                                   EVT VT, bool isT, bool isO) {
    1111             :   assert(VT.isInteger() && "Cannot create FP integer constant!");
    1112             : 
    1113     8999477 :   EVT EltVT = VT.getScalarType();
    1114     8999477 :   const ConstantInt *Elt = &Val;
    1115             : 
    1116             :   // In some cases the vector type is legal but the element type is illegal and
    1117             :   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
    1118             :   // inserted value (the type does not need to match the vector element type).
    1119             :   // Any extra bits introduced will be truncated away.
    1120     9135689 :   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
    1121             :       TargetLowering::TypePromoteInteger) {
    1122       13408 :    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
    1123       20112 :    APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
    1124        6704 :    Elt = ConstantInt::get(*getContext(), NewVal);
    1125             :   }
    1126             :   // In other cases the element type is illegal and needs to be expanded, for
    1127             :   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
    1128             :   // the value into n parts and use a vector type with n-times the elements.
    1129             :   // Then bitcast to the type requested.
    1130             :   // Legalizing constants too early makes the DAGCombiner's job harder so we
    1131             :   // only legalize if the DAG tells us we must produce legal types.
    1132    14294206 :   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
    1133       89870 :            TLI->getTypeAction(*getContext(), EltVT) ==
    1134             :            TargetLowering::TypeExpandInteger) {
    1135         370 :     const APInt &NewVal = Elt->getValue();
    1136         370 :     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
    1137         185 :     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
    1138         185 :     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
    1139         185 :     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
    1140             : 
    1141             :     // Check the temporary vector is the correct size. If this fails then
    1142             :     // getTypeToTransformTo() probably returned a type whose size (in bits)
    1143             :     // isn't a power-of-2 factor of the requested type size.
    1144             :     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
    1145             : 
    1146         370 :     SmallVector<SDValue, 2> EltParts;
    1147         555 :     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
    1148        1110 :       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
    1149         740 :                                            .zextOrTrunc(ViaEltSizeInBits), DL,
    1150         370 :                                      ViaEltVT, isT, isO));
    1151             :     }
    1152             : 
    1153             :     // EltParts is currently in little endian order. If we actually want
    1154             :     // big-endian order then reverse it now.
    1155         370 :     if (getDataLayout().isBigEndian())
    1156         116 :       std::reverse(EltParts.begin(), EltParts.end());
    1157             : 
    1158             :     // The elements must be reversed when the element order is different
    1159             :     // to the endianness of the elements (because the BITCAST is itself a
    1160             :     // vector shuffle in this situation). However, we do not need any code to
    1161             :     // perform this reversal because getConstant() is producing a vector
    1162             :     // splat.
    1163             :     // This situation occurs in MIPS MSA.
    1164             : 
    1165         370 :     SmallVector<SDValue, 8> Ops;
    1166         741 :     for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    1167        1668 :       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
    1168             : 
    1169         370 :     SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
    1170         185 :     NewSDValueDbgMsg(V, "Creating constant: ");
    1171         185 :     return V;
    1172             :   }
    1173             : 
    1174             :   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
    1175             :          "APInt size does not match type size!");
    1176     8999292 :   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
    1177     8999292 :   FoldingSetNodeID ID;
    1178     8999292 :   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
    1179     8999292 :   ID.AddPointer(Elt);
    1180    17998584 :   ID.AddBoolean(isO);
    1181     8999292 :   void *IP = nullptr;
    1182     8999292 :   SDNode *N = nullptr;
    1183     8999292 :   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
    1184     5898887 :     if (!VT.isVector())
    1185     5857656 :       return SDValue(N, 0);
    1186             : 
    1187     3141636 :   if (!N) {
    1188     3100405 :     N = newSDNode<ConstantSDNode>(isT, isO, Elt, DL.getDebugLoc(), EltVT);
    1189     6200810 :     CSEMap.InsertNode(N, IP);
    1190     3100405 :     InsertNode(N);
    1191             :   }
    1192             : 
    1193     3141636 :   SDValue Result(N, 0);
    1194     3141636 :   if (VT.isVector())
    1195      136027 :     Result = getSplatBuildVector(VT, DL, Result);
    1196             : 
    1197     3141636 :   NewSDValueDbgMsg(Result, "Creating constant: ");
    1198     3141636 :   return Result;
    1199             : }
    1200             : 
    1201      998192 : SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
    1202             :                                         bool isTarget) {
    1203     3992768 :   return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
    1204             : }
    1205             : 
    1206        9590 : SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
    1207             :                                     bool isTarget) {
    1208        9590 :   return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
    1209             : }
    1210             : 
    1211       24347 : SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
    1212             :                                     EVT VT, bool isTarget) {
    1213             :   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
    1214             : 
    1215       24347 :   EVT EltVT = VT.getScalarType();
    1216             : 
    1217             :   // Do the map lookup using the actual bit pattern for the floating point
    1218             :   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
    1219             :   // we don't have issues with SNANs.
    1220       24347 :   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
    1221       48694 :   FoldingSetNodeID ID;
    1222       24347 :   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
    1223       24347 :   ID.AddPointer(&V);
    1224       24347 :   void *IP = nullptr;
    1225       24347 :   SDNode *N = nullptr;
    1226       24347 :   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
    1227        4114 :     if (!VT.isVector())
    1228        3717 :       return SDValue(N, 0);
    1229             : 
    1230       20630 :   if (!N) {
    1231       20233 :     N = newSDNode<ConstantFPSDNode>(isTarget, &V, DL.getDebugLoc(), EltVT);
    1232       40466 :     CSEMap.InsertNode(N, IP);
    1233       20233 :     InsertNode(N);
    1234             :   }
    1235             : 
    1236       20630 :   SDValue Result(N, 0);
    1237       20630 :   if (VT.isVector())
    1238         913 :     Result = getSplatBuildVector(VT, DL, Result);
    1239       20630 :   NewSDValueDbgMsg(Result, "Creating fp constant: ");
    1240       20630 :   return Result;
    1241             : }
    1242             : 
    1243        4636 : SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
    1244             :                                     bool isTarget) {
    1245        4636 :   EVT EltVT = VT.getScalarType();
    1246        9272 :   if (EltVT == MVT::f32)
    1247        6228 :     return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
    1248        3044 :   else if (EltVT == MVT::f64)
    1249        2706 :     return getConstantFP(APFloat(Val), DL, VT, isTarget);
    1250         480 :   else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
    1251         150 :            EltVT == MVT::f16) {
    1252             :     bool Ignored;
    1253         338 :     APFloat APF = APFloat(Val);
    1254         169 :     APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
    1255             :                 &Ignored);
    1256         169 :     return getConstantFP(APF, DL, VT, isTarget);
    1257             :   } else
    1258           0 :     llvm_unreachable("Unsupported type in getConstantFP");
    1259             : }
    1260             : 
    1261     1648961 : SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
    1262             :                                        EVT VT, int64_t Offset, bool isTargetGA,
    1263             :                                        unsigned char TargetFlags) {
    1264             :   assert((TargetFlags == 0 || isTargetGA) &&
    1265             :          "Cannot set target flags on target-independent globals");
    1266             : 
    1267             :   // Truncate (with sign-extension) the offset value to the pointer size.
    1268     4946883 :   unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
    1269     1648961 :   if (BitWidth < 64)
    1270     1717088 :     Offset = SignExtend64(Offset, BitWidth);
    1271             : 
    1272             :   unsigned Opc;
    1273     3297922 :   if (GV->isThreadLocal())
    1274        1673 :     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
    1275             :   else
    1276     1647288 :     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
    1277             : 
    1278     3297922 :   FoldingSetNodeID ID;
    1279     1648961 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1280     1648961 :   ID.AddPointer(GV);
    1281     1648961 :   ID.AddInteger(Offset);
    1282     1648961 :   ID.AddInteger(TargetFlags);
    1283     1648961 :   void *IP = nullptr;
    1284     1648961 :   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    1285      259374 :     return SDValue(E, 0);
    1286             : 
    1287     1389587 :   auto *N = newSDNode<GlobalAddressSDNode>(
    1288     2779174 :       Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
    1289     2779174 :   CSEMap.InsertNode(N, IP);
    1290     1389587 :     InsertNode(N);
    1291     1389587 :   return SDValue(N, 0);
    1292             : }
    1293             : 
    1294      322238 : SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
    1295      322238 :   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
    1296      644476 :   FoldingSetNodeID ID;
    1297      322238 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1298      322238 :   ID.AddInteger(FI);
    1299      322238 :   void *IP = nullptr;
    1300      322238 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1301       53147 :     return SDValue(E, 0);
    1302             : 
    1303      269091 :   auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
    1304      538182 :   CSEMap.InsertNode(N, IP);
    1305      269091 :   InsertNode(N);
    1306      269091 :   return SDValue(N, 0);
    1307             : }
    1308             : 
    1309         531 : SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
    1310             :                                    unsigned char TargetFlags) {
    1311             :   assert((TargetFlags == 0 || isTarget) &&
    1312             :          "Cannot set target flags on target-independent jump tables");
    1313         531 :   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
    1314        1062 :   FoldingSetNodeID ID;
    1315         531 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1316         531 :   ID.AddInteger(JTI);
    1317         531 :   ID.AddInteger(TargetFlags);
    1318         531 :   void *IP = nullptr;
    1319         531 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1320          45 :     return SDValue(E, 0);
    1321             : 
    1322         486 :   auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
    1323         972 :   CSEMap.InsertNode(N, IP);
    1324         486 :   InsertNode(N);
    1325         486 :   return SDValue(N, 0);
    1326             : }
    1327             : 
    1328       55779 : SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
    1329             :                                       unsigned Alignment, int Offset,
    1330             :                                       bool isTarget,
    1331             :                                       unsigned char TargetFlags) {
    1332             :   assert((TargetFlags == 0 || isTarget) &&
    1333             :          "Cannot set target flags on target-independent globals");
    1334       55779 :   if (Alignment == 0)
    1335       19698 :     Alignment = MF->getFunction()->optForSize()
    1336       39396 :                     ? getDataLayout().getABITypeAlignment(C->getType())
    1337       39226 :                     : getDataLayout().getPrefTypeAlignment(C->getType());
    1338       55779 :   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
    1339      111558 :   FoldingSetNodeID ID;
    1340       55779 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1341       55779 :   ID.AddInteger(Alignment);
    1342       55779 :   ID.AddInteger(Offset);
    1343       55779 :   ID.AddPointer(C);
    1344       55779 :   ID.AddInteger(TargetFlags);
    1345       55779 :   void *IP = nullptr;
    1346       55779 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1347        3420 :     return SDValue(E, 0);
    1348             : 
    1349             :   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
    1350       52359 :                                           TargetFlags);
    1351      104718 :   CSEMap.InsertNode(N, IP);
    1352       52359 :   InsertNode(N);
    1353       52359 :   return SDValue(N, 0);
    1354             : }
    1355             : 
    1356         353 : SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
    1357             :                                       unsigned Alignment, int Offset,
    1358             :                                       bool isTarget,
    1359             :                                       unsigned char TargetFlags) {
    1360             :   assert((TargetFlags == 0 || isTarget) &&
    1361             :          "Cannot set target flags on target-independent globals");
    1362         353 :   if (Alignment == 0)
    1363           0 :     Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
    1364         353 :   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
    1365         706 :   FoldingSetNodeID ID;
    1366         353 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1367         353 :   ID.AddInteger(Alignment);
    1368         353 :   ID.AddInteger(Offset);
    1369         353 :   C->addSelectionDAGCSEId(ID);
    1370         353 :   ID.AddInteger(TargetFlags);
    1371         353 :   void *IP = nullptr;
    1372         353 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1373           0 :     return SDValue(E, 0);
    1374             : 
    1375             :   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
    1376         353 :                                           TargetFlags);
    1377         706 :   CSEMap.InsertNode(N, IP);
    1378         353 :   InsertNode(N);
    1379         353 :   return SDValue(N, 0);
    1380             : }
    1381             : 
    1382           0 : SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
    1383             :                                      unsigned char TargetFlags) {
    1384           0 :   FoldingSetNodeID ID;
    1385           0 :   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
    1386           0 :   ID.AddInteger(Index);
    1387           0 :   ID.AddInteger(Offset);
    1388           0 :   ID.AddInteger(TargetFlags);
    1389           0 :   void *IP = nullptr;
    1390           0 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1391           0 :     return SDValue(E, 0);
    1392             : 
    1393           0 :   auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
    1394           0 :   CSEMap.InsertNode(N, IP);
    1395           0 :   InsertNode(N);
    1396           0 :   return SDValue(N, 0);
    1397             : }
    1398             : 
    1399      134617 : SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
    1400      269234 :   FoldingSetNodeID ID;
    1401      269234 :   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
    1402      134617 :   ID.AddPointer(MBB);
    1403      134617 :   void *IP = nullptr;
    1404      134617 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1405           5 :     return SDValue(E, 0);
    1406             : 
    1407      134612 :   auto *N = newSDNode<BasicBlockSDNode>(MBB);
    1408      269224 :   CSEMap.InsertNode(N, IP);
    1409      134612 :   InsertNode(N);
    1410      134612 :   return SDValue(N, 0);
    1411             : }
    1412             : 
    1413       76145 : SDValue SelectionDAG::getValueType(EVT VT) {
    1414      150439 :   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
    1415       74294 :       ValueTypeNodes.size())
    1416        3971 :     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
    1417             : 
    1418      226584 :   SDNode *&N = VT.isExtended() ?
    1419      152290 :     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
    1420             : 
    1421       76145 :   if (N) return SDValue(N, 0);
    1422       40970 :   N = newSDNode<VTSDNode>(VT);
    1423       40970 :   InsertNode(N);
    1424       40970 :   return SDValue(N, 0);
    1425             : }
    1426             : 
    1427        7345 : SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
    1428       22035 :   SDNode *&N = ExternalSymbols[Sym];
    1429        7345 :   if (N) return SDValue(N, 0);
    1430        5307 :   N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
    1431        5307 :   InsertNode(N);
    1432        5307 :   return SDValue(N, 0);
    1433             : }
    1434             : 
    1435          64 : SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
    1436         128 :   SDNode *&N = MCSymbols[Sym];
    1437          64 :   if (N)
    1438          16 :     return SDValue(N, 0);
    1439          48 :   N = newSDNode<MCSymbolSDNode>(Sym, VT);
    1440          48 :   InsertNode(N);
    1441          48 :   return SDValue(N, 0);
    1442             : }
    1443             : 
    1444       18758 : SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
    1445             :                                               unsigned char TargetFlags) {
    1446             :   SDNode *&N =
    1447       37516 :     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
    1448       18758 :                                                                TargetFlags)];
    1449       18758 :   if (N) return SDValue(N, 0);
    1450       12596 :   N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
    1451       12596 :   InsertNode(N);
    1452       12596 :   return SDValue(N, 0);
    1453             : }
    1454             : 
    1455      144395 : SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
    1456      288790 :   if ((unsigned)Cond >= CondCodeNodes.size())
    1457        7609 :     CondCodeNodes.resize(Cond+1);
    1458             : 
    1459      288790 :   if (!CondCodeNodes[Cond]) {
    1460       96670 :     auto *N = newSDNode<CondCodeSDNode>(Cond);
    1461      193340 :     CondCodeNodes[Cond] = N;
    1462       96670 :     InsertNode(N);
    1463             :   }
    1464             : 
    1465      288790 :   return SDValue(CondCodeNodes[Cond], 0);
    1466             : }
    1467             : 
    1468             : /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
    1469             : /// point at N1 to point at N2 and indices that point at N2 to point at N1.
    1470             : static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
    1471         628 :   std::swap(N1, N2);
    1472             :   ShuffleVectorSDNode::commuteMask(M);
    1473             : }
    1474             : 
    1475       71931 : SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
    1476             :                                        SDValue N2, ArrayRef<int> Mask) {
    1477             :   assert(VT.getVectorNumElements() == Mask.size() &&
    1478             :            "Must have the same number of vector elements as mask elements!");
    1479             :   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
    1480             :          "Invalid VECTOR_SHUFFLE");
    1481             : 
    1482             :   // Canonicalize shuffle undef, undef -> undef
    1483      144043 :   if (N1.isUndef() && N2.isUndef())
    1484          59 :     return getUNDEF(VT);
    1485             : 
    1486             :   // Validate that all indices in Mask are within the range of the elements
    1487             :   // input to the shuffle.
    1488       71872 :   int NElts = Mask.size();
    1489             :   assert(llvm::all_of(Mask, [&](int M) { return M < (NElts * 2); }) &&
    1490             :          "Index out of range");
    1491             : 
    1492             :   // Copy the mask so we can do any needed cleanup.
    1493      143744 :   SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
    1494             : 
    1495             :   // Canonicalize shuffle v, v -> v, undef
    1496        3267 :   if (N1 == N2) {
    1497        3267 :     N2 = getUNDEF(VT);
    1498       29026 :     for (int i = 0; i != NElts; ++i)
    1499       53086 :       if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
    1500             :   }
    1501             : 
    1502             :   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
    1503      143744 :   if (N1.isUndef())
    1504         122 :     commuteShuffle(N1, N2, MaskVec);
    1505             : 
    1506             :   // If shuffling a splat, try to blend the splat instead. We do this here so
    1507             :   // that even when this arises during lowering we don't have to re-handle it.
    1508        9340 :   auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
    1509       17897 :     BitVector UndefElements;
    1510        9340 :     SDValue Splat = BV->getSplatValue(&UndefElements);
    1511        9340 :     if (!Splat)
    1512         783 :       return;
    1513             : 
    1514      218339 :     for (int i = 0; i < NElts; ++i) {
    1515      291204 :       if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
    1516       34035 :         continue;
    1517             : 
    1518             :       // If this input comes from undef, mark it as such.
    1519      283449 :       if (UndefElements[MaskVec[i] - Offset]) {
    1520          50 :         MaskVec[i] = -1;
    1521          25 :         continue;
    1522             :       }
    1523             : 
    1524             :       // If we can blend a non-undef lane, use that instead.
    1525      212493 :       if (!UndefElements[i])
    1526      138806 :         MaskVec[i] = i + Offset;
    1527             :     }
    1528       71872 :   };
    1529        4239 :   if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
    1530        4239 :     BlendSplat(N1BV, 0);
    1531        5101 :   if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
    1532        5101 :     BlendSplat(N2BV, NElts);
    1533             : 
    1534             :   // Canonicalize all index into lhs, -> shuffle lhs, undef
    1535             :   // Canonicalize all index into rhs, -> shuffle rhs, undef
    1536       71872 :   bool AllLHS = true, AllRHS = true;
    1537      143744 :   bool N2Undef = N2.isUndef();
    1538      756958 :   for (int i = 0; i != NElts; ++i) {
    1539     1370172 :     if (MaskVec[i] >= NElts) {
    1540      127062 :       if (N2Undef)
    1541        5524 :         MaskVec[i] = -1;
    1542             :       else
    1543             :         AllLHS = false;
    1544     1116048 :     } else if (MaskVec[i] >= 0) {
    1545      387299 :       AllRHS = false;
    1546             :     }
    1547             :   }
    1548       71872 :   if (AllLHS && AllRHS)
    1549          45 :     return getUNDEF(VT);
    1550       71827 :   if (AllLHS && !N2Undef)
    1551        2414 :     N2 = getUNDEF(VT);
    1552       71827 :   if (AllRHS) {
    1553         506 :     N1 = getUNDEF(VT);
    1554         506 :     commuteShuffle(N1, N2, MaskVec);
    1555             :   }
    1556             :   // Reset our undef status after accounting for the mask.
    1557      143654 :   N2Undef = N2.isUndef();
    1558             :   // Re-check whether both sides ended up undef.
    1559      143654 :   if (N1.isUndef() && N2Undef)
    1560           0 :     return getUNDEF(VT);
    1561             : 
    1562             :   // If Identity shuffle return that node.
    1563             :   bool Identity = true, AllSame = true;
    1564     1441625 :   for (int i = 0; i != NElts; ++i) {
    1565     1881397 :     if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
    1566     2054697 :     if (MaskVec[i] != MaskVec[0]) AllSame = false;
    1567             :   }
    1568       71827 :   if (Identity && NElts)
    1569        7780 :     return N1;
    1570             : 
    1571             :   // Shuffling a constant splat doesn't change the result.
    1572       64047 :   if (N2Undef) {
    1573             :     SDValue V = N1;
    1574             : 
    1575             :     // Look through any bitcasts. We check that these don't change the number
    1576             :     // (and size) of elements and just changes their types.
    1577       89921 :     while (V.getOpcode() == ISD::BITCAST)
    1578       18354 :       V = V->getOperand(0);
    1579             : 
    1580             :     // A splat should always show up as a build vector node.
    1581         844 :     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
    1582        1424 :       BitVector UndefElements;
    1583         844 :       SDValue Splat = BV->getSplatValue(&UndefElements);
    1584             :       // If this is a splat of an undef, shuffling it is also undef.
    1585         956 :       if (Splat && Splat.isUndef())
    1586         264 :         return getUNDEF(VT);
    1587             : 
    1588             :       bool SameNumElts =
    1589        1688 :           V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
    1590             : 
    1591             :       // We only have a splat which can skip shuffles if there is a splatted
    1592             :       // value and no undef lanes rearranged by the shuffle.
    1593         956 :       if (Splat && UndefElements.none()) {
    1594             :         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
    1595             :         // number of elements match or the value splatted is a zero constant.
    1596          25 :         if (SameNumElts)
    1597           4 :           return N1;
    1598          20 :         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
    1599          20 :           if (C->isNullValue())
    1600          13 :             return N1;
    1601             :       }
    1602             : 
    1603             :       // If the shuffle itself creates a splat, build the vector directly.
    1604         827 :       if (AllSame && SameNumElts) {
    1605         494 :         EVT BuildVT = BV->getValueType(0);
    1606         741 :         const SDValue &Splatted = BV->getOperand(MaskVec[0]);
    1607         247 :         SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
    1608             : 
    1609             :         // We may have jumped through bitcasts, so the type of the
    1610             :         // BUILD_VECTOR may not match the type of the shuffle.
    1611         247 :         if (BuildVT != VT)
    1612           0 :           NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
    1613         247 :         return NewBV;
    1614             :       }
    1615             :     }
    1616             :   }
    1617             : 
    1618       63783 :   FoldingSetNodeID ID;
    1619       63783 :   SDValue Ops[2] = { N1, N2 };
    1620       63783 :   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
    1621      689105 :   for (int i = 0; i != NElts; ++i)
    1622     1250644 :     ID.AddInteger(MaskVec[i]);
    1623             : 
    1624       63783 :   void* IP = nullptr;
    1625       63783 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
    1626        1033 :     return SDValue(E, 0);
    1627             : 
    1628             :   // Allocate the mask array for the node out of the BumpPtrAllocator, since
    1629             :   // SDNode doesn't have access to it.  This memory will be "leaked" when
    1630             :   // the node is deallocated, but recovered when the NodeAllocator is released.
    1631      125500 :   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
    1632      251000 :   std::copy(MaskVec.begin(), MaskVec.end(), MaskAlloc);
    1633             : 
    1634      125500 :   auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
    1635       62750 :                                            dl.getDebugLoc(), MaskAlloc);
    1636       62750 :   createOperands(N, Ops);
    1637             : 
    1638      125500 :   CSEMap.InsertNode(N, IP);
    1639       62750 :   InsertNode(N);
    1640       62750 :   return SDValue(N, 0);
    1641             : }
    1642             : 
    1643        4002 : SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
    1644        8004 :   MVT VT = SV.getSimpleValueType(0);
    1645       20010 :   SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
    1646        8004 :   ShuffleVectorSDNode::commuteMask(MaskVec);
    1647             : 
    1648        8004 :   SDValue Op0 = SV.getOperand(0);
    1649        8004 :   SDValue Op1 = SV.getOperand(1);
    1650       20010 :   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
    1651             : }
    1652             : 
    1653     4349247 : SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
    1654     8698494 :   FoldingSetNodeID ID;
    1655     4349247 :   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
    1656     4349247 :   ID.AddInteger(RegNo);
    1657     4349247 :   void *IP = nullptr;
    1658     4349247 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1659     2982171 :     return SDValue(E, 0);
    1660             : 
    1661     1367076 :   auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
    1662     2734152 :   CSEMap.InsertNode(N, IP);
    1663     1367076 :   InsertNode(N);
    1664     1367076 :   return SDValue(N, 0);
    1665             : }
    1666             : 
    1667      188153 : SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
    1668      376306 :   FoldingSetNodeID ID;
    1669      376306 :   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
    1670      188153 :   ID.AddPointer(RegMask);
    1671      188153 :   void *IP = nullptr;
    1672      188153 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1673       92289 :     return SDValue(E, 0);
    1674             : 
    1675       95864 :   auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
    1676      191728 :   CSEMap.InsertNode(N, IP);
    1677       95864 :   InsertNode(N);
    1678       95864 :   return SDValue(N, 0);
    1679             : }
    1680             : 
    1681       53258 : SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
    1682             :                                  MCSymbol *Label) {
    1683       53258 :   return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
    1684             : }
    1685             : 
    1686       53259 : SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
    1687             :                                    SDValue Root, MCSymbol *Label) {
    1688      106518 :   FoldingSetNodeID ID;
    1689       53259 :   SDValue Ops[] = { Root };
    1690      106518 :   AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
    1691       53259 :   ID.AddPointer(Label);
    1692       53259 :   void *IP = nullptr;
    1693       53259 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1694           0 :     return SDValue(E, 0);
    1695             : 
    1696      106518 :   auto *N = newSDNode<LabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label);
    1697       53259 :   createOperands(N, Ops);
    1698             : 
    1699      106518 :   CSEMap.InsertNode(N, IP);
    1700       53259 :   InsertNode(N);
    1701       53259 :   return SDValue(N, 0);
    1702             : }
    1703             : 
    1704         243 : SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
    1705             :                                       int64_t Offset,
    1706             :                                       bool isTarget,
    1707             :                                       unsigned char TargetFlags) {
    1708         243 :   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
    1709             : 
    1710         486 :   FoldingSetNodeID ID;
    1711         243 :   AddNodeIDNode(ID, Opc, getVTList(VT), None);
    1712         243 :   ID.AddPointer(BA);
    1713         243 :   ID.AddInteger(Offset);
    1714         243 :   ID.AddInteger(TargetFlags);
    1715         243 :   void *IP = nullptr;
    1716         243 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1717           2 :     return SDValue(E, 0);
    1718             : 
    1719         241 :   auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
    1720         482 :   CSEMap.InsertNode(N, IP);
    1721         241 :   InsertNode(N);
    1722         241 :   return SDValue(N, 0);
    1723             : }
    1724             : 
    1725         534 : SDValue SelectionDAG::getSrcValue(const Value *V) {
    1726             :   assert((!V || V->getType()->isPointerTy()) &&
    1727             :          "SrcValue is not a pointer?");
    1728             : 
    1729        1068 :   FoldingSetNodeID ID;
    1730        1068 :   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
    1731         534 :   ID.AddPointer(V);
    1732             : 
    1733         534 :   void *IP = nullptr;
    1734         534 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1735         168 :     return SDValue(E, 0);
    1736             : 
    1737         366 :   auto *N = newSDNode<SrcValueSDNode>(V);
    1738         732 :   CSEMap.InsertNode(N, IP);
    1739         366 :   InsertNode(N);
    1740         366 :   return SDValue(N, 0);
    1741             : }
    1742             : 
    1743        9325 : SDValue SelectionDAG::getMDNode(const MDNode *MD) {
    1744       18650 :   FoldingSetNodeID ID;
    1745       18650 :   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
    1746        9325 :   ID.AddPointer(MD);
    1747             : 
    1748        9325 :   void *IP = nullptr;
    1749        9325 :   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
    1750           0 :     return SDValue(E, 0);
    1751             : 
    1752        9325 :   auto *N = newSDNode<MDNodeSDNode>(MD);
    1753       18650 :   CSEMap.InsertNode(N, IP);
    1754        9325 :   InsertNode(N);
    1755        9325 :   return SDValue(N, 0);
    1756             : }
    1757             : 
    1758      242424 : SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
    1759      484882 :   if (VT == V.getValueType())
    1760       56911 :     return V;
    1761             : 
    1762      556539 :   return getNode(ISD::BITCAST, SDLoc(V), VT, V);
    1763             : }
    1764             : 
    1765         199 : SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
    1766             :                                        unsigned SrcAS, unsigned DestAS) {
    1767         199 :   SDValue Ops[] = {Ptr};
    1768         398 :   FoldingSetNodeID ID;
    1769         199 :   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
    1770         199 :   ID.AddInteger(SrcAS);
    1771         199 :   ID.AddInteger(DestAS);
    1772             : 
    1773         199 :   void *IP = nullptr;
    1774         199 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
    1775           0 :     return SDValue(E, 0);
    1776             : 
    1777         398 :   auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
    1778         199 :                                            VT, SrcAS, DestAS);
    1779         199 :   createOperands(N, Ops);
    1780             : 
    1781         398 :   CSEMap.InsertNode(N, IP);
    1782         199 :   InsertNode(N);
    1783         199 :   return SDValue(N, 0);
    1784             : }
    1785             : 
    1786             : /// getShiftAmountOperand - Return the specified value casted to
    1787             : /// the target's desired shift amount type.
    1788      256445 : SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
    1789      512890 :   EVT OpTy = Op.getValueType();
    1790      512890 :   EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
    1791      512890 :   if (OpTy == ShTy || OpTy.isVector()) return Op;
    1792             : 
    1793       75596 :   return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
    1794             : }
    1795             : 
    1796          27 : SDValue SelectionDAG::expandVAArg(SDNode *Node) {
    1797          54 :   SDLoc dl(Node);
    1798          27 :   const TargetLowering &TLI = getTargetLoweringInfo();
    1799          81 :   const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
    1800          54 :   EVT VT = Node->getValueType(0);
    1801          54 :   SDValue Tmp1 = Node->getOperand(0);
    1802          54 :   SDValue Tmp2 = Node->getOperand(1);
    1803          27 :   unsigned Align = Node->getConstantOperandVal(3);
    1804             : 
    1805             :   SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
    1806         135 :                                Tmp2, MachinePointerInfo(V));
    1807          27 :   SDValue VAList = VAListLoad;
    1808             : 
    1809          27 :   if (Align > TLI.getMinStackArgumentAlignment()) {
    1810             :     assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
    1811             : 
    1812          11 :     VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
    1813          33 :                      getConstant(Align - 1, dl, VAList.getValueType()));
    1814             : 
    1815          11 :     VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
    1816          33 :                      getConstant(-(int64_t)Align, dl, VAList.getValueType()));
    1817             :   }
    1818             : 
    1819             :   // Increment the pointer, VAList, to the next vaarg
    1820          27 :   Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
    1821          27 :                  getConstant(getDataLayout().getTypeAllocSize(
    1822          27 :                                                VT.getTypeForEVT(*getContext())),
    1823          81 :                              dl, VAList.getValueType()));
    1824             :   // Store the incremented VAList to the legalized pointer
    1825          27 :   Tmp1 =
    1826         108 :       getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
    1827             :   // Load the actual argument out of the pointer VAList
    1828          81 :   return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
    1829             : }
    1830             : 
    1831           5 : SDValue SelectionDAG::expandVACopy(SDNode *Node) {
    1832          10 :   SDLoc dl(Node);
    1833           5 :   const TargetLowering &TLI = getTargetLoweringInfo();
    1834             :   // This defaults to loading a pointer from the input and storing it to the
    1835             :   // output, returning the chain.
    1836          15 :   const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
    1837          15 :   const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
    1838             :   SDValue Tmp1 =
    1839          10 :       getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
    1840          35 :               Node->getOperand(2), MachinePointerInfo(VS));
    1841          10 :   return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
    1842          25 :                   MachinePointerInfo(VD));
    1843             : }
    1844             : 
    1845        2433 : SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
    1846        2433 :   MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
    1847        2433 :   unsigned ByteSize = VT.getStoreSize();
    1848        2433 :   Type *Ty = VT.getTypeForEVT(*getContext());
    1849             :   unsigned StackAlign =
    1850        7299 :       std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
    1851             : 
    1852        2433 :   int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
    1853        7299 :   return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
    1854             : }
    1855             : 
    1856        3733 : SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
    1857       11199 :   unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
    1858        3733 :   Type *Ty1 = VT1.getTypeForEVT(*getContext());
    1859        3733 :   Type *Ty2 = VT2.getTypeForEVT(*getContext());
    1860        7466 :   const DataLayout &DL = getDataLayout();
    1861             :   unsigned Align =
    1862        7466 :       std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
    1863             : 
    1864        3733 :   MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
    1865        3733 :   int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
    1866       11199 :   return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
    1867             : }
    1868             : 
    1869      186866 : SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
    1870             :                                 ISD::CondCode Cond, const SDLoc &dl) {
    1871             :   // These setcc operations always fold.
    1872      186866 :   switch (Cond) {
    1873             :   default: break;
    1874          37 :   case ISD::SETFALSE:
    1875          37 :   case ISD::SETFALSE2: return getConstant(0, dl, VT);
    1876          42 :   case ISD::SETTRUE:
    1877             :   case ISD::SETTRUE2: {
    1878             :     TargetLowering::BooleanContent Cnt =
    1879          84 :         TLI->getBooleanContents(N1->getValueType(0));
    1880             :     return getConstant(
    1881             :         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, dl,
    1882          42 :         VT);
    1883             :   }
    1884             : 
    1885             :   case ISD::SETOEQ:
    1886             :   case ISD::SETOGT:
    1887             :   case ISD::SETOGE:
    1888             :   case ISD::SETOLT:
    1889             :   case ISD::SETOLE:
    1890             :   case ISD::SETONE:
    1891             :   case ISD::SETO:
    1892             :   case ISD::SETUO:
    1893             :   case ISD::SETUEQ:
    1894             :   case ISD::SETUNE:
    1895             :     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
    1896             :     break;
    1897             :   }
    1898             : 
    1899      115523 :   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
    1900      115523 :     const APInt &C2 = N2C->getAPIntValue();
    1901        1737 :     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
    1902        1737 :       const APInt &C1 = N1C->getAPIntValue();
    1903             : 
    1904        1737 :       switch (Cond) {
    1905           0 :       default: llvm_unreachable("Unknown integer setcc!");
    1906         428 :       case ISD::SETEQ:  return getConstant(C1 == C2, dl, VT);
    1907         240 :       case ISD::SETNE:  return getConstant(C1 != C2, dl, VT);
    1908         358 :       case ISD::SETULT: return getConstant(C1.ult(C2), dl, VT);
    1909         348 :       case ISD::SETUGT: return getConstant(C1.ugt(C2), dl, VT);
    1910         388 :       case ISD::SETULE: return getConstant(C1.ule(C2), dl, VT);
    1911         344 :       case ISD::SETUGE: return getConstant(C1.uge(C2), dl, VT);
    1912         392 :       case ISD::SETLT:  return getConstant(C1.slt(C2), dl, VT);
    1913         384 :       case ISD::SETGT:  return getConstant(C1.sgt(C2), dl, VT);
    1914         286 :       case ISD::SETLE:  return getConstant(C1.sle(C2), dl, VT);
    1915         306 :       case ISD::SETGE:  return getConstant(C1.sge(C2), dl, VT);
    1916             :       }
    1917             :     }
    1918             :   }
    1919         449 :   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) {
    1920         230 :     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) {
    1921         230 :       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
    1922         230 :       switch (Cond) {
    1923             :       default: break;
    1924           4 :       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
    1925           0 :                           return getUNDEF(VT);
    1926             :                         LLVM_FALLTHROUGH;
    1927          91 :       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT);
    1928           4 :       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
    1929           0 :                           return getUNDEF(VT);
    1930             :                         LLVM_FALLTHROUGH;
    1931           8 :       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
    1932           8 :                                            R==APFloat::cmpLessThan, dl, VT);
    1933           0 :       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
    1934           0 :                           return getUNDEF(VT);
    1935             :                         LLVM_FALLTHROUGH;
    1936          22 :       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT);
    1937           0 :       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
    1938           0 :                           return getUNDEF(VT);
    1939             :                         LLVM_FALLTHROUGH;
    1940           2 :       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT);
    1941           0 :       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
    1942           0 :                           return getUNDEF(VT);
    1943             :                         LLVM_FALLTHROUGH;
    1944           4 :       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
    1945           4 :                                            R==APFloat::cmpEqual, dl, VT);
    1946           0 :       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
    1947           0 :                           return getUNDEF(VT);
    1948             :                         LLVM_FALLTHROUGH;
    1949           5 :       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
    1950           5 :                                            R==APFloat::cmpEqual, dl, VT);
    1951           8 :       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, dl, VT);
    1952          31 :       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, dl, VT);
    1953           0 :       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
    1954           0 :                                            R==APFloat::cmpEqual, dl, VT);
    1955          18 :       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, dl, VT);
    1956          12 :       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
    1957          12 :                                            R==APFloat::cmpLessThan, dl, VT);
    1958          36 :       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
    1959          36 :                                            R==APFloat::cmpUnordered, dl, VT);
    1960           1 :       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, dl, VT);
    1961           2 :       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, dl, VT);
    1962             :       }
    1963             :     } else {
    1964             :       // Ensure that the constant occurs on the RHS.
    1965         219 :       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
    1966         438 :       MVT CompVT = N1.getValueType().getSimpleVT();
    1967         322 :       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
    1968         103 :         return SDValue();
    1969             : 
    1970         116 :       return getSetCC(dl, VT, N2, N1, SwappedCond);
    1971             :     }
    1972             :   }
    1973             : 
    1974             :   // Could not fold it.
    1975      184601 :   return SDValue();
    1976             : }
    1977             : 
    1978             : /// See if the specified operand can be simplified with the knowledge that only
    1979             : /// the bits specified by Mask are used.
    1980      214835 : SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) {
    1981      429670 :   switch (V.getOpcode()) {
    1982             :   default:
    1983             :     break;
    1984        4205 :   case ISD::Constant: {
    1985        8410 :     const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode());
    1986             :     assert(CV && "Const value should be ConstSDNode.");
    1987        4205 :     const APInt &CVal = CV->getAPIntValue();
    1988       12615 :     APInt NewVal = CVal & Mask;
    1989        4205 :     if (NewVal != CVal)
    1990         616 :       return getConstant(NewVal, SDLoc(V), V.getValueType());
    1991        4051 :     break;
    1992             :   }
    1993        3032 :   case ISD::OR:
    1994             :   case ISD::XOR:
    1995             :     // If the LHS or RHS don't contribute bits to the or, drop them.
    1996        6064 :     if (MaskedValueIsZero(V.getOperand(0), Mask))
    1997        1116 :       return V.getOperand(1);
    1998        4948 :     if (MaskedValueIsZero(V.getOperand(1), Mask))
    1999        1324 :       return V.getOperand(0);
    2000             :     break;
    2001       39222 :   case ISD::SRL:
    2002             :     // Only look at single-use SRLs.
    2003       75872 :     if (!V.getNode()->hasOneUse())
    2004             :       break;
    2005      109831 :     if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
    2006             :       // See if we can recursively simplify the LHS.
    2007       36531 :       unsigned Amt = RHSC->getZExtValue();
    2008             : 
    2009             :       // Watch out for shift count overflow though.
    2010       36531 :       if (Amt >= Mask.getBitWidth())
    2011             :         break;
    2012       72038 :       APInt NewMask = Mask << Amt;
    2013       73060 :       if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask))
    2014        2044 :         return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
    2015        5110 :                        V.getOperand(1));
    2016             :     }
    2017             :     break;
    2018        9348 :   case ISD::AND: {
    2019             :     // X & -1 -> X (ignoring bits which aren't demanded).
    2020       18696 :     ConstantSDNode *AndVal = isConstOrConstSplat(V.getOperand(1));
    2021       27502 :     if (AndVal && Mask.isSubsetOf(AndVal->getAPIntValue()))
    2022        2610 :       return V.getOperand(0);
    2023             :     break;
    2024             :   }
    2025         742 :   case ISD::ANY_EXTEND: {
    2026        1484 :     SDValue Src = V.getOperand(0);
    2027         742 :     unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
    2028             :     // Being conservative here - only peek through if we only demand bits in the
    2029             :     // non-extended source (even though the extended bits are technically undef).
    2030         742 :     if (Mask.getActiveBits() > SrcBitWidth)
    2031             :       break;
    2032         740 :     APInt SrcMask = Mask.trunc(SrcBitWidth);
    2033         740 :     if (SDValue DemandedSrc = GetDemandedBits(Src, SrcMask))
    2034         244 :       return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
    2035             :     break;
    2036             :   }
    2037             :   }
    2038      211073 :   return SDValue();
    2039             : }
    2040             : 
    2041             : /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
    2042             : /// use this predicate to simplify operations downstream.
    2043       46880 : bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
    2044       46880 :   unsigned BitWidth = Op.getScalarValueSizeInBits();
    2045       93760 :   return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
    2046             : }
    2047             : 
    2048             : /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
    2049             : /// this predicate to simplify operations downstream.  Mask is known to be zero
    2050             : /// for bits that V cannot have.
    2051     1355161 : bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
    2052             :                                      unsigned Depth) const {
    2053     2710322 :   KnownBits Known;
    2054     1355161 :   computeKnownBits(Op, Known, Depth);
    2055     2710322 :   return Mask.isSubsetOf(Known.Zero);
    2056             : }
    2057             : 
    2058             : /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
    2059             : /// is less than the element bit-width of the shift node, return it.
    2060     1429004 : static const APInt *getValidShiftAmountConstant(SDValue V) {
    2061     2858008 :   if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
    2062             :     // Shifting more than the bitwidth is not valid.
    2063     2803472 :     const APInt &ShAmt = SA->getAPIntValue();
    2064     1401736 :     if (ShAmt.ult(V.getScalarValueSizeInBits()))
    2065             :       return &ShAmt;
    2066             :   }
    2067             :   return nullptr;
    2068             : }
    2069             : 
    2070             : /// Determine which bits of Op are known to be either zero or one and return
    2071             : /// them in Known. For vectors, the known bits are those that are shared by
    2072             : /// every vector element.
    2073    11969016 : void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known,
    2074             :                                     unsigned Depth) const {
    2075    23938032 :   EVT VT = Op.getValueType();
    2076    11969016 :   APInt DemandedElts = VT.isVector()
    2077             :                            ? APInt::getAllOnesValue(VT.getVectorNumElements())
    2078    23938032 :                            : APInt(1, 1);
    2079    11969016 :   computeKnownBits(Op, Known, DemandedElts, Depth);
    2080    11969016 : }
    2081             : 
    2082             : /// Determine which bits of Op are known to be either zero or one and return
    2083             : /// them in Known. The DemandedElts argument allows us to only collect the known
    2084             : /// bits that are shared by the requested vector elements.
    2085    23389128 : void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known,
    2086             :                                     const APInt &DemandedElts,
    2087             :                                     unsigned Depth) const {
    2088    23389128 :   unsigned BitWidth = Op.getScalarValueSizeInBits();
    2089             : 
    2090    23389128 :   Known = KnownBits(BitWidth);   // Don't know anything.
    2091    23389128 :   if (Depth == 6)
    2092      584695 :     return;  // Limit search depth.
    2093             : 
    2094    45608866 :   KnownBits Known2;
    2095    22804433 :   unsigned NumElts = DemandedElts.getBitWidth();
    2096             : 
    2097    22804433 :   if (!DemandedElts)
    2098           0 :     return;  // No demanded elts, better to assume we don't know anything.
    2099             : 
    2100    45608866 :   unsigned Opcode = Op.getOpcode();
    2101    22804433 :   switch (Opcode) {
    2102     4605762 :   case ISD::Constant:
    2103             :     // We know all of the bits for a constant!
    2104     9211524 :     Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
    2105    27634572 :     Known.Zero = ~Known.One;
    2106     4605762 :     break;
    2107      354077 :   case ISD::BUILD_VECTOR:
    2108             :     // Collect the known bits that are shared by every demanded vector element.
    2109             :     assert(NumElts == Op.getValueType().getVectorNumElements() &&
    2110             :            "Unexpected vector size");
    2111      354077 :     Known.Zero.setAllBits(); Known.One.setAllBits();
    2112     2481661 :     for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
    2113     1885284 :       if (!DemandedElts[i])
    2114     1136750 :         continue;
    2115             : 
    2116     1497068 :       SDValue SrcOp = Op.getOperand(i);
    2117      748534 :       computeKnownBits(SrcOp, Known2, Depth + 1);
    2118             : 
    2119             :       // BUILD_VECTOR can implicitly truncate sources, we must handle this.
    2120      748534 :       if (SrcOp.getValueSizeInBits() != BitWidth) {
    2121             :         assert(SrcOp.getValueSizeInBits() > BitWidth &&
    2122             :                "Expected BUILD_VECTOR implicit truncation");
    2123       15303 :         Known2 = Known2.trunc(BitWidth);
    2124             :       }
    2125             : 
    2126             :       // Known bits are the values that are shared by every demanded element.
    2127     1497068 :       Known.One &= Known2.One;
    2128     1497068 :       Known.Zero &= Known2.Zero;
    2129             : 
    2130             :       // If we don't know any bits, early out.
    2131     1745366 :       if (!Known.One && !Known.Zero)
    2132             :         break;
    2133      242300 :     }
    2134             :     break;
    2135       23595 :   case ISD::VECTOR_SHUFFLE: {
    2136             :     // Collect the known bits that are shared by every vector element referenced
    2137             :     // by the shuffle.
    2138       94380 :     APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
    2139       23595 :     Known.Zero.setAllBits(); Known.One.setAllBits();
    2140       23595 :     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
    2141             :     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
    2142      283818 :     for (unsigned i = 0; i != NumElts; ++i) {
    2143      264126 :       if (!DemandedElts[i])
    2144      140811 :         continue;
    2145             : 
    2146      246630 :       int M = SVN->getMaskElt(i);
    2147      123315 :       if (M < 0) {
    2148             :         // For UNDEF elements, we don't know anything about the common state of
    2149             :         // the shuffle result.
    2150        3903 :         Known.resetAll();
    2151        3903 :         DemandedLHS.clearAllBits();
    2152        3903 :         DemandedRHS.clearAllBits();
    2153        3903 :         break;
    2154             :       }
    2155             : 
    2156      119412 :       if ((unsigned)M < NumElts)
    2157       83080 :         DemandedLHS.setBit((unsigned)M % NumElts);
    2158             :       else
    2159       36332 :         DemandedRHS.setBit((unsigned)M % NumElts);
    2160             :     }
    2161             :     // Known bits are the values that are shared by every demanded element.
    2162       23595 :     if (!!DemandedLHS) {
    2163       35422 :       SDValue LHS = Op.getOperand(0);
    2164       17711 :       computeKnownBits(LHS, Known2, DemandedLHS, Depth + 1);
    2165       35422 :       Known.One &= Known2.One;
    2166       35422 :       Known.Zero &= Known2.Zero;
    2167             :     }
    2168             :     // If we don't know any bits, early out.
    2169       68785 :     if (!Known.One && !Known.Zero)
    2170             :       break;
    2171        2674 :     if (!!DemandedRHS) {
    2172        4172 :       SDValue RHS = Op.getOperand(1);
    2173        2086 :       computeKnownBits(RHS, Known2, DemandedRHS, Depth + 1);
    2174        4172 :       Known.One &= Known2.One;
    2175        4172 :       Known.Zero &= Known2.Zero;
    2176             :     }
    2177             :     break;
    2178             :   }
    2179       11638 :   case ISD::CONCAT_VECTORS: {
    2180             :     // Split DemandedElts and test each of the demanded subvectors.
    2181       11638 :     Known.Zero.setAllBits(); Known.One.setAllBits();
    2182       34914 :     EVT SubVectorVT = Op.getOperand(0).getValueType();
    2183       11638 :     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
    2184       23276 :     unsigned NumSubVectors = Op.getNumOperands();
    2185       17438 :     for (unsigned i = 0; i != NumSubVectors; ++i) {
    2186       22186 :       APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
    2187       49158 :       DemandedSub = DemandedSub.trunc(NumSubVectorElts);
    2188       16386 :       if (!!DemandedSub) {
    2189       26968 :         SDValue Sub = Op.getOperand(i);
    2190       13484 :         computeKnownBits(Sub, Known2, DemandedSub, Depth + 1);
    2191       26968 :         Known.One &= Known2.One;
    2192       26968 :         Known.Zero &= Known2.Zero;
    2193             :       }
    2194             :       // If we don't know any bits, early out.
    2195       46267 :       if (!Known.One && !Known.Zero)
    2196             :         break;
    2197             :     }
    2198             :     break;
    2199             :   }
    2200       39884 :   case ISD::EXTRACT_SUBVECTOR: {
    2201             :     // If we know the element index, just demand that subvector elements,
    2202             :     // otherwise demand them all.
    2203       79768 :     SDValue Src = Op.getOperand(0);
    2204      119652 :     ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
    2205       79768 :     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
    2206      119652 :     if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
    2207             :       // Offset the demanded elts by the subvector index.
    2208       39884 :       uint64_t Idx = SubIdx->getZExtValue();
    2209      119652 :       APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx);
    2210       39884 :       computeKnownBits(Src, Known, DemandedSrc, Depth + 1);
    2211             :     } else {
    2212           0 :       computeKnownBits(Src, Known, Depth + 1);
    2213             :     }
    2214             :     break;
    2215             :   }
    2216      447895 :   case ISD::BITCAST: {
    2217      895790 :     SDValue N0 = Op.getOperand(0);
    2218      447895 :     unsigned SubBitWidth = N0.getScalarValueSizeInBits();
    2219             : 
    2220             :     // Ignore bitcasts from floating point.
    2221      895790 :     if (!N0.getValueType().isInteger())
    2222             :       break;
    2223             : 
    2224             :     // Fast handling of 'identity' bitcasts.
    2225      420078 :     if (BitWidth == SubBitWidth) {
    2226         144 :       computeKnownBits(N0, Known, DemandedElts, Depth + 1);
    2227         144 :       break;
    2228             :     }
    2229             : 
    2230             :     // Support big-endian targets when it becomes useful.
    2231      839868 :     bool IsLE = getDataLayout().isLittleEndian();
    2232      419934 :     if (!IsLE)
    2233             :       break;
    2234             : 
    2235             :     // Bitcast 'small element' vector to 'large element' scalar/vector.
    2236      415837 :     if ((BitWidth % SubBitWidth) == 0) {
    2237             :       assert(N0.getValueType().isVector() && "Expected bitcast from vector");
    2238             : 
    2239             :       // Collect known bits for the (larger) output by collecting the known
    2240             :       // bits from each set of sub elements and shift these into place.
    2241             :       // We need to separately call computeKnownBits for each set of
    2242             :       // sub elements as the knownbits for each is likely to be different.
    2243      228580 :       unsigned SubScale = BitWidth / SubBitWidth;
    2244      457160 :       APInt SubDemandedElts(NumElts * SubScale, 0);
    2245     1093684 :       for (unsigned i = 0; i != NumElts; ++i)
    2246      432552 :         if (DemandedElts[i])
    2247      398060 :           SubDemandedElts.setBit(i * SubScale);
    2248             : 
    2249     1895264 :       for (unsigned i = 0; i != SubScale; ++i) {
    2250     1666684 :         computeKnownBits(N0, Known2, SubDemandedElts.shl(i),
    2251             :                          Depth + 1);
    2252     3333368 :         Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * i);
    2253     3333368 :         Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * i);
    2254             :       }
    2255             :     }
    2256             : 
    2257             :     // Bitcast 'large element' scalar/vector to 'small element' vector.
    2258      415837 :     if ((SubBitWidth % BitWidth) == 0) {
    2259             :       assert(Op.getValueType().isVector() && "Expected bitcast to vector");
    2260             : 
    2261             :       // Collect known bits for the (smaller) output by collecting the known
    2262             :       // bits from the overlapping larger input elements and extracting the
    2263             :       // sub sections we actually care about.
    2264      187257 :       unsigned SubScale = SubBitWidth / BitWidth;
    2265      374514 :       APInt SubDemandedElts(NumElts / SubScale, 0);
    2266    10407051 :       for (unsigned i = 0; i != NumElts; ++i)
    2267     5109897 :         if (DemandedElts[i])
    2268      528377 :           SubDemandedElts.setBit(i / SubScale);
    2269             : 
    2270      187257 :       computeKnownBits(N0, Known2, SubDemandedElts, Depth + 1);
    2271             : 
    2272      187257 :       Known.Zero.setAllBits(); Known.One.setAllBits();
    2273     2331806 :       for (unsigned i = 0; i != NumElts; ++i)
    2274     2325765 :         if (DemandedElts[i]) {
    2275      222405 :           unsigned Offset = (i % SubScale) * BitWidth;
    2276      889620 :           Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
    2277      889620 :           Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
    2278             :           // If we don't know any bits, early out.
    2279      661587 :           if (!Known.One && !Known.Zero)
    2280             :             break;
    2281             :         }
    2282             :     }
    2283             :     break;
    2284             :   }
    2285      528037 :   case ISD::AND:
    2286             :     // If either the LHS or the RHS are Zero, the result is zero.
    2287     1056074 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2288     1056074 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2289             : 
    2290             :     // Output known-1 bits are only known if set in both the LHS & RHS.
    2291     1056074 :     Known.One &= Known2.One;
    2292             :     // Output known-0 are known to be clear if zero in either the LHS | RHS.
    2293      528037 :     Known.Zero |= Known2.Zero;
    2294             :     break;
    2295      117135 :   case ISD::OR:
    2296      234270 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2297      234270 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2298             : 
    2299             :     // Output known-0 bits are only known if clear in both the LHS & RHS.
    2300      234270 :     Known.Zero &= Known2.Zero;
    2301             :     // Output known-1 are known to be set if set in either the LHS | RHS.
    2302      117135 :     Known.One |= Known2.One;
    2303             :     break;
    2304      151596 :   case ISD::XOR: {
    2305      303192 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2306      303192 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2307             : 
    2308             :     // Output known-0 bits are known if clear or set in both the LHS & RHS.
    2309     1667556 :     APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
    2310             :     // Output known-1 are known to be set if set in only one of the LHS, RHS.
    2311     1819152 :     Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
    2312      151596 :     Known.Zero = KnownZeroOut;
    2313             :     break;
    2314             :   }
    2315       69424 :   case ISD::MUL: {
    2316      138848 :     computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
    2317      138848 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2318             : 
    2319             :     // If low bits are zero in either operand, output low known-0 bits.
    2320             :     // Also compute a conservative estimate for high known-0 bits.
    2321             :     // More trickiness is possible, but this is sufficient for the
    2322             :     // interesting case of alignment computation.
    2323      138848 :     unsigned TrailZ = Known.countMinTrailingZeros() +
    2324      138848 :                       Known2.countMinTrailingZeros();
    2325      277696 :     unsigned LeadZ =  std::max(Known.countMinLeadingZeros() +
    2326       69424 :                                Known2.countMinLeadingZeros(),
    2327      138848 :                                BitWidth) - BitWidth;
    2328             : 
    2329       69424 :     Known.resetAll();
    2330      208272 :     Known.Zero.setLowBits(std::min(TrailZ, BitWidth));
    2331      208272 :     Known.Zero.setHighBits(std::min(LeadZ, BitWidth));
    2332             :     break;
    2333             :   }
    2334        1060 :   case ISD::UDIV: {
    2335             :     // For the purposes of computing leading zeros we can conservatively
    2336             :     // treat a udiv as a logical right shift by the power of 2 known to
    2337             :     // be less than the denominator.
    2338        2120 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2339        1060 :     unsigned LeadZ = Known2.countMinLeadingZeros();
    2340             : 
    2341        2120 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2342        1060 :     unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
    2343        1060 :     if (RHSMaxLeadingZeros != BitWidth)
    2344        1016 :       LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
    2345             : 
    2346        1060 :     Known.Zero.setHighBits(LeadZ);
    2347             :     break;
    2348             :   }
    2349      107359 :   case ISD::SELECT:
    2350      214718 :     computeKnownBits(Op.getOperand(2), Known, Depth+1);
    2351             :     // If we don't know any bits, early out.
    2352      226088 :     if (!Known.One && !Known.Zero)
    2353             :       break;
    2354      199244 :     computeKnownBits(Op.getOperand(1), Known2, Depth+1);
    2355             : 
    2356             :     // Only known if known in both the LHS and RHS.
    2357      199244 :     Known.One &= Known2.One;
    2358       99622 :     Known.Zero &= Known2.Zero;
    2359             :     break;
    2360       35613 :   case ISD::SELECT_CC:
    2361       71226 :     computeKnownBits(Op.getOperand(3), Known, Depth+1);
    2362             :     // If we don't know any bits, early out.
    2363      106800 :     if (!Known.One && !Known.Zero)
    2364             :       break;
    2365       10856 :     computeKnownBits(Op.getOperand(2), Known2, Depth+1);
    2366             : 
    2367             :     // Only known if known in both the LHS and RHS.
    2368       10856 :     Known.One &= Known2.One;
    2369        5428 :     Known.Zero &= Known2.Zero;
    2370             :     break;
    2371         175 :   case ISD::SMULO:
    2372             :   case ISD::UMULO:
    2373         175 :     if (Op.getResNo() != 1)
    2374             :       break;
    2375             :     // The boolean result conforms to getBooleanContents.
    2376             :     // If we know the result of a setcc has the top bits zero, use this info.
    2377             :     // We know that we have an integer-based boolean since these operations
    2378             :     // are only available for integer.
    2379         576 :     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
    2380         143 :             TargetLowering::ZeroOrOneBooleanContent &&
    2381             :         BitWidth > 1)
    2382         139 :       Known.Zero.setBitsFrom(1);
    2383             :     break;
    2384      543641 :   case ISD::SETCC:
    2385             :     // If we know the result of a setcc has the top bits zero, use this info.
    2386     1630923 :     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    2387      543641 :             TargetLowering::ZeroOrOneBooleanContent &&
    2388             :         BitWidth > 1)
    2389      300931 :       Known.Zero.setBitsFrom(1);
    2390             :     break;
    2391      878358 :   case ISD::SHL:
    2392      878358 :     if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
    2393     1724434 :       computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2394      862217 :       Known.Zero <<= *ShAmt;
    2395      862217 :       Known.One <<= *ShAmt;
    2396             :       // Low bits are known zero.
    2397     1724434 :       Known.Zero.setLowBits(ShAmt->getZExtValue());
    2398             :     }
    2399             :     break;
    2400      515720 :   case ISD::SRL:
    2401      515720 :     if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
    2402     1011278 :       computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2403      505639 :       Known.Zero.lshrInPlace(*ShAmt);
    2404      505639 :       Known.One.lshrInPlace(*ShAmt);
    2405             :       // High bits are known zero.
    2406     1011278 :       Known.Zero.setHighBits(ShAmt->getZExtValue());
    2407             :     }
    2408             :     break;
    2409       34926 :   case ISD::SRA:
    2410       34926 :     if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
    2411       67716 :       computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2412       33858 :       Known.Zero.lshrInPlace(*ShAmt);
    2413       33858 :       Known.One.lshrInPlace(*ShAmt);
    2414             :       // If we know the value of the sign bit, then we know it is copied across
    2415             :       // the high bits by the shift amount.
    2416      101574 :       APInt SignMask = APInt::getSignMask(BitWidth);
    2417       33858 :       SignMask.lshrInPlace(*ShAmt);  // Adjust to where it is now in the mask.
    2418       67716 :       if (Known.Zero.intersects(SignMask)) {
    2419          20 :         Known.Zero.setHighBits(ShAmt->getZExtValue());// New bits are known zero.
    2420       67696 :       } else if (Known.One.intersects(SignMask)) {
    2421           2 :         Known.One.setHighBits(ShAmt->getZExtValue()); // New bits are known one.
    2422             :       }
    2423             :     }
    2424             :     break;
    2425       28499 :   case ISD::SIGN_EXTEND_INREG: {
    2426       85497 :     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
    2427       28499 :     unsigned EBits = EVT.getScalarSizeInBits();
    2428             : 
    2429             :     // Sign extension.  Compute the demanded bits in the result that are not
    2430             :     // present in the input.
    2431       56998 :     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
    2432             : 
    2433       56998 :     APInt InSignMask = APInt::getSignMask(EBits);
    2434       56998 :     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
    2435             : 
    2436             :     // If the sign extended bits are demanded, we know that the sign
    2437             :     // bit is demanded.
    2438       85497 :     InSignMask = InSignMask.zext(BitWidth);
    2439       28499 :     if (NewBits.getBoolValue())
    2440             :       InputDemandedBits |= InSignMask;
    2441             : 
    2442       56998 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2443       56998 :     Known.One &= InputDemandedBits;
    2444       56998 :     Known.Zero &= InputDemandedBits;
    2445             : 
    2446             :     // If the sign bit of the input is known set or clear, then we know the
    2447             :     // top bits of the result.
    2448       56998 :     if (Known.Zero.intersects(InSignMask)) {        // Input sign bit known clear
    2449          16 :       Known.Zero |= NewBits;
    2450          40 :       Known.One  &= ~NewBits;
    2451       56982 :     } else if (Known.One.intersects(InSignMask)) {  // Input sign bit known set
    2452           4 :       Known.One  |= NewBits;
    2453          10 :       Known.Zero &= ~NewBits;
    2454             :     } else {                              // Input sign bit unknown
    2455      142445 :       Known.Zero &= ~NewBits;
    2456      142445 :       Known.One  &= ~NewBits;
    2457             :     }
    2458             :     break;
    2459             :   }
    2460         341 :   case ISD::CTTZ:
    2461             :   case ISD::CTTZ_ZERO_UNDEF: {
    2462         682 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2463             :     // If we have a known 1, its position is our upper bound.
    2464         341 :     unsigned PossibleTZ = Known2.countMaxTrailingZeros();
    2465         341 :     unsigned LowBits = Log2_32(PossibleTZ) + 1;
    2466         341 :     Known.Zero.setBitsFrom(LowBits);
    2467             :     break;
    2468             :   }
    2469        2086 :   case ISD::CTLZ:
    2470             :   case ISD::CTLZ_ZERO_UNDEF: {
    2471        4172 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2472             :     // If we have a known 1, its position is our upper bound.
    2473        2086 :     unsigned PossibleLZ = Known2.countMaxLeadingZeros();
    2474        2086 :     unsigned LowBits = Log2_32(PossibleLZ) + 1;
    2475        2086 :     Known.Zero.setBitsFrom(LowBits);
    2476             :     break;
    2477             :   }
    2478        1549 :   case ISD::CTPOP: {
    2479        3098 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2480             :     // If we know some of the bits are zero, they can't be one.
    2481        1549 :     unsigned PossibleOnes = Known2.countMaxPopulation();
    2482        3098 :     Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
    2483             :     break;
    2484             :   }
    2485     2071996 :   case ISD::LOAD: {
    2486     2071996 :     LoadSDNode *LD = cast<LoadSDNode>(Op);
    2487             :     // If this is a ZEXTLoad and we are looking at the loaded value.
    2488     2134501 :     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
    2489       62427 :       EVT VT = LD->getMemoryVT();
    2490       62427 :       unsigned MemBits = VT.getScalarSizeInBits();
    2491      124854 :       Known.Zero.setBitsFrom(MemBits);
    2492     4019138 :     } else if (const MDNode *Ranges = LD->getRanges()) {
    2493         839 :       if (LD->getExtensionType() == ISD::NON_EXTLOAD)
    2494         833 :         computeKnownBitsFromRangeMetadata(*Ranges, Known);
    2495             :     }
    2496             :     break;
    2497             :   }
    2498        3971 :   case ISD::ZERO_EXTEND_VECTOR_INREG: {
    2499       11913 :     EVT InVT = Op.getOperand(0).getValueType();
    2500        3971 :     unsigned InBits = InVT.getScalarSizeInBits();
    2501        3971 :     Known = Known.trunc(InBits);
    2502       11913 :     computeKnownBits(Op.getOperand(0), Known,
    2503        7942 :                      DemandedElts.zext(InVT.getVectorNumElements()),
    2504             :                      Depth + 1);
    2505        3971 :     Known = Known.zext(BitWidth);
    2506        7942 :     Known.Zero.setBitsFrom(InBits);
    2507             :     break;
    2508             :   }
    2509      557538 :   case ISD::ZERO_EXTEND: {
    2510     1672614 :     EVT InVT = Op.getOperand(0).getValueType();
    2511      557538 :     unsigned InBits = InVT.getScalarSizeInBits();
    2512      557538 :     Known = Known.trunc(InBits);
    2513     1115076 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2514      557538 :     Known = Known.zext(BitWidth);
    2515     1115076 :     Known.Zero.setBitsFrom(InBits);
    2516             :     break;
    2517             :   }
    2518             :   // TODO ISD::SIGN_EXTEND_VECTOR_INREG
    2519       52423 :   case ISD::SIGN_EXTEND: {
    2520      157269 :     EVT InVT = Op.getOperand(0).getValueType();
    2521       52423 :     unsigned InBits = InVT.getScalarSizeInBits();
    2522             : 
    2523       52423 :     Known = Known.trunc(InBits);
    2524      104846 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2525             : 
    2526             :     // If the sign bit is known to be zero or one, then sext will extend
    2527             :     // it to the top bits, else it will just zext.
    2528       52423 :     Known = Known.sext(BitWidth);
    2529             :     break;
    2530             :   }
    2531      119078 :   case ISD::ANY_EXTEND: {
    2532      357234 :     EVT InVT = Op.getOperand(0).getValueType();
    2533      119078 :     unsigned InBits = InVT.getScalarSizeInBits();
    2534      119078 :     Known = Known.trunc(InBits);
    2535      238156 :     computeKnownBits(Op.getOperand(0), Known, Depth+1);
    2536      119078 :     Known = Known.zext(BitWidth);
    2537             :     break;
    2538             :   }
    2539      341992 :   case ISD::TRUNCATE: {
    2540     1025976 :     EVT InVT = Op.getOperand(0).getValueType();
    2541      341992 :     unsigned InBits = InVT.getScalarSizeInBits();
    2542      341992 :     Known = Known.zext(InBits);
    2543      683984 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2544      341992 :     Known = Known.trunc(BitWidth);
    2545             :     break;
    2546             :   }
    2547      268581 :   case ISD::AssertZext: {
    2548      805743 :     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
    2549      537162 :     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
    2550      537162 :     computeKnownBits(Op.getOperand(0), Known, Depth+1);
    2551     1342905 :     Known.Zero |= (~InMask);
    2552     1611486 :     Known.One  &= (~Known.Zero);
    2553             :     break;
    2554             :   }
    2555          26 :   case ISD::FGETSIGN:
    2556             :     // All bits are zero except the low bit.
    2557          26 :     Known.Zero.setBitsFrom(1);
    2558             :     break;
    2559        4540 :   case ISD::USUBO:
    2560             :   case ISD::SSUBO:
    2561        4540 :     if (Op.getResNo() == 1) {
    2562             :       // If we know the result of a setcc has the top bits zero, use this info.
    2563       13149 :       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    2564        4383 :               TargetLowering::ZeroOrOneBooleanContent &&
    2565             :           BitWidth > 1)
    2566         111 :         Known.Zero.setBitsFrom(1);
    2567             :       break;
    2568             :     }
    2569             :     LLVM_FALLTHROUGH;
    2570             :   case ISD::SUB:
    2571             :   case ISD::SUBC: {
    2572      134558 :     if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0))) {
    2573             :       // We know that the top bits of C-X are clear if X contains less bits
    2574             :       // than C (i.e. no wrap-around can happen).  For example, 20-X is
    2575             :       // positive if we can prove that X is >= 0 and < 16.
    2576       23577 :       if (CLHS->getAPIntValue().isNonNegative()) {
    2577       45186 :         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
    2578             :         // NLZ can't be BitWidth with no sign bit
    2579       15062 :         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
    2580       15062 :         computeKnownBits(Op.getOperand(1), Known2, DemandedElts,
    2581             :                          Depth + 1);
    2582             : 
    2583             :         // If all of the MaskV bits are known to be zero, then we know the
    2584             :         // output top bits are zero, because we now know that the output is
    2585             :         // from [0-C].
    2586       37655 :         if ((Known2.Zero & MaskV) == MaskV) {
    2587        4014 :           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
    2588             :           // Top bits known zero.
    2589        2007 :           Known.Zero.setHighBits(NLZ2);
    2590             :         }
    2591             :       }
    2592             :     }
    2593             : 
    2594             :     // If low bits are know to be zero in both operands, then we know they are
    2595             :     // going to be 0 in the result. Both addition and complement operations
    2596             :     // preserve the low zero bits.
    2597      134558 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2598       67279 :     unsigned KnownZeroLow = Known2.countMinTrailingZeros();
    2599       67279 :     if (KnownZeroLow == 0)
    2600             :       break;
    2601             : 
    2602       12382 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2603       12382 :     KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
    2604        6191 :     Known.Zero.setLowBits(KnownZeroLow);
    2605             :     break;
    2606             :   }
    2607        5968 :   case ISD::UADDO:
    2608             :   case ISD::SADDO:
    2609             :   case ISD::ADDCARRY:
    2610        5968 :     if (Op.getResNo() == 1) {
    2611             :       // If we know the result of a setcc has the top bits zero, use this info.
    2612        3147 :       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    2613        1049 :               TargetLowering::ZeroOrOneBooleanContent &&
    2614             :           BitWidth > 1)
    2615         822 :         Known.Zero.setBitsFrom(1);
    2616             :       break;
    2617             :     }
    2618             :     LLVM_FALLTHROUGH;
    2619             :   case ISD::ADD:
    2620             :   case ISD::ADDC:
    2621             :   case ISD::ADDE: {
    2622             :     // Output known-0 bits are known if clear or set in both the low clear bits
    2623             :     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
    2624             :     // low 3 bits clear.
    2625             :     // Output known-0 bits are also known if the top bits of each input are
    2626             :     // known to be clear. For example, if one input has the top 10 bits clear
    2627             :     // and the other has the top 8 bits clear, we know the top 7 bits of the
    2628             :     // output must be clear.
    2629     5582876 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2630     2791438 :     unsigned KnownZeroHigh = Known2.countMinLeadingZeros();
    2631     2791438 :     unsigned KnownZeroLow = Known2.countMinTrailingZeros();
    2632             : 
    2633     5582876 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts,
    2634             :                      Depth + 1);
    2635     5582876 :     KnownZeroHigh = std::min(KnownZeroHigh, Known2.countMinLeadingZeros());
    2636     5582876 :     KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
    2637             : 
    2638     2791438 :     if (Opcode == ISD::ADDE || Opcode == ISD::ADDCARRY) {
    2639             :       // With ADDE and ADDCARRY, a carry bit may be added in, so we can only
    2640             :       // use this information if we know (at least) that the low two bits are
    2641             :       // clear. We then return to the caller that the low bit is unknown but
    2642             :       // that other bits are known zero.
    2643        1184 :       if (KnownZeroLow >= 2)
    2644         109 :         Known.Zero.setBits(1, KnownZeroLow);
    2645             :       break;
    2646             :     }
    2647             : 
    2648     5580508 :     Known.Zero.setLowBits(KnownZeroLow);
    2649     2790254 :     if (KnownZeroHigh > 1)
    2650       25069 :       Known.Zero.setHighBits(KnownZeroHigh - 1);
    2651             :     break;
    2652             :   }
    2653        1254 :   case ISD::SREM:
    2654        2508 :     if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
    2655         682 :       const APInt &RA = Rem->getAPIntValue().abs();
    2656         341 :       if (RA.isPowerOf2()) {
    2657          76 :         APInt LowBits = RA - 1;
    2658          38 :         computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2659             : 
    2660             :         // The low bits of the first operand are unchanged by the srem.
    2661          95 :         Known.Zero = Known2.Zero & LowBits;
    2662          95 :         Known.One = Known2.One & LowBits;
    2663             : 
    2664             :         // If the first operand is non-negative or has all low bits zero, then
    2665             :         // the upper bits are all zero.
    2666         133 :         if (Known2.Zero[BitWidth-1] || ((Known2.Zero & LowBits) == LowBits))
    2667          10 :           Known.Zero |= ~LowBits;
    2668             : 
    2669             :         // If the first operand is negative and not all low bits are zero, then
    2670             :         // the upper bits are all one.
    2671          76 :         if (Known2.One[BitWidth-1] && ((Known2.One & LowBits) != 0))
    2672           0 :           Known.One |= ~LowBits;
    2673             :         assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?");
    2674             :       }
    2675             :     }
    2676             :     break;
    2677        1442 :   case ISD::UREM: {
    2678        2884 :     if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
    2679        1498 :       const APInt &RA = Rem->getAPIntValue();
    2680         749 :       if (RA.isPowerOf2()) {
    2681          32 :         APInt LowBits = (RA - 1);
    2682          16 :         computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2683             : 
    2684             :         // The upper bits are all zero, the lower ones are unchanged.
    2685          56 :         Known.Zero = Known2.Zero | ~LowBits;
    2686          40 :         Known.One = Known2.One & LowBits;
    2687             :         break;
    2688             :       }
    2689             :     }
    2690             : 
    2691             :     // Since the result is less than or equal to either operand, any leading
    2692             :     // zero bits in either operand must also exist in the result.
    2693        2868 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2694        2868 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2695             : 
    2696             :     uint32_t Leaders =
    2697        4302 :         std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
    2698        1434 :     Known.resetAll();
    2699        1434 :     Known.Zero.setHighBits(Leaders);
    2700             :     break;
    2701             :   }
    2702          50 :   case ISD::EXTRACT_ELEMENT: {
    2703         100 :     computeKnownBits(Op.getOperand(0), Known, Depth+1);
    2704         100 :     const unsigned Index = Op.getConstantOperandVal(1);
    2705          50 :     const unsigned BitWidth = Op.getValueSizeInBits();
    2706             : 
    2707             :     // Remove low part of known bits mask
    2708         150 :     Known.Zero = Known.Zero.getHiBits(Known.Zero.getBitWidth() - Index * BitWidth);
    2709         150 :     Known.One = Known.One.getHiBits(Known.One.getBitWidth() - Index * BitWidth);
    2710             : 
    2711             :     // Remove high part of known bit mask
    2712          50 :     Known = Known.trunc(BitWidth);
    2713          50 :     break;
    2714             :   }
    2715      313835 :   case ISD::EXTRACT_VECTOR_ELT: {
    2716      627670 :     SDValue InVec = Op.getOperand(0);
    2717      627670 :     SDValue EltNo = Op.getOperand(1);
    2718      627670 :     EVT VecVT = InVec.getValueType();
    2719      313835 :     const unsigned BitWidth = Op.getValueSizeInBits();
    2720      313835 :     const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
    2721      313835 :     const unsigned NumSrcElts = VecVT.getVectorNumElements();
    2722             :     // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
    2723             :     // anything about the extended bits.
    2724      313835 :     if (BitWidth > EltBitWidth)
    2725       10109 :       Known = Known.trunc(EltBitWidth);
    2726      313300 :     ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
    2727      313300 :     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) {
    2728             :       // If we know the element index, just demand that vector element.
    2729      313300 :       unsigned Idx = ConstEltNo->getZExtValue();
    2730      626600 :       APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
    2731      313300 :       computeKnownBits(InVec, Known, DemandedElt, Depth + 1);
    2732             :     } else {
    2733             :       // Unknown element index, so ignore DemandedElts and demand them all.
    2734         535 :       computeKnownBits(InVec, Known, Depth + 1);
    2735             :     }
    2736      313835 :     if (BitWidth > EltBitWidth)
    2737       10109 :       Known = Known.zext(BitWidth);
    2738             :     break;
    2739             :   }
    2740        3511 :   case ISD::INSERT_VECTOR_ELT: {
    2741        7022 :     SDValue InVec = Op.getOperand(0);
    2742        7022 :     SDValue InVal = Op.getOperand(1);
    2743        7022 :     SDValue EltNo = Op.getOperand(2);
    2744             : 
    2745        3507 :     ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
    2746        3507 :     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
    2747             :       // If we know the element index, split the demand between the
    2748             :       // source vector and the inserted element.
    2749       10521 :       Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth);
    2750        3507 :       unsigned EltIdx = CEltNo->getZExtValue();
    2751             : 
    2752             :       // If we demand the inserted element then add its common known bits.
    2753        3507 :       if (DemandedElts[EltIdx]) {
    2754        2451 :         computeKnownBits(InVal, Known2, Depth + 1);
    2755        7353 :         Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
    2756        7353 :         Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
    2757             :       }
    2758             : 
    2759             :       // If we demand the source vector then add its common known bits, ensuring
    2760             :       // that we don't demand the inserted element.
    2761       21042 :       APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
    2762        3507 :       if (!!VectorElts) {
    2763        2581 :         computeKnownBits(InVec, Known2, VectorElts, Depth + 1);
    2764        5162 :         Known.One &= Known2.One;
    2765        2581 :         Known.Zero &= Known2.Zero;
    2766             :       }
    2767             :     } else {
    2768             :       // Unknown element index, so ignore DemandedElts and demand them all.
    2769           4 :       computeKnownBits(InVec, Known, Depth + 1);
    2770           4 :       computeKnownBits(InVal, Known2, Depth + 1);
    2771          12 :       Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
    2772          12 :       Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
    2773             :     }
    2774             :     break;
    2775             :   }
    2776         121 :   case ISD::BITREVERSE: {
    2777         242 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2778         363 :     Known.Zero = Known2.Zero.reverseBits();
    2779         363 :     Known.One = Known2.One.reverseBits();
    2780         121 :     break;
    2781             :   }
    2782        1689 :   case ISD::BSWAP: {
    2783        3378 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2784        5067 :     Known.Zero = Known2.Zero.byteSwap();
    2785        5067 :     Known.One = Known2.One.byteSwap();
    2786        1689 :     break;
    2787             :   }
    2788         692 :   case ISD::ABS: {
    2789        1384 :     computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
    2790             : 
    2791             :     // If the source's MSB is zero then we know the rest of the bits already.
    2792         692 :     if (Known2.isNonNegative()) {
    2793           0 :       Known.Zero = Known2.Zero;
    2794           0 :       Known.One = Known2.One;
    2795           0 :       break;
    2796             :     }
    2797             : 
    2798             :     // We only know that the absolute values's MSB will be zero iff there is
    2799             :     // a set bit that isn't the sign bit (otherwise it could be INT_MIN).
    2800        1384 :     Known2.One.clearSignBit();
    2801         692 :     if (Known2.One.getBoolValue()) {
    2802           8 :       Known.Zero = APInt::getSignMask(BitWidth);
    2803           2 :       break;
    2804             :     }
    2805             :     break;
    2806             :   }
    2807         626 :   case ISD::UMIN: {
    2808        1252 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
    2809        1252 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2810             : 
    2811             :     // UMIN - we know that the result will have the maximum of the
    2812             :     // known zero leading bits of the inputs.
    2813         626 :     unsigned LeadZero = Known.countMinLeadingZeros();
    2814        1252 :     LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros());
    2815             : 
    2816        1252 :     Known.Zero &= Known2.Zero;
    2817        1252 :     Known.One &= Known2.One;
    2818        1252 :     Known.Zero.setHighBits(LeadZero);
    2819             :     break;
    2820             :   }
    2821         303 :   case ISD::UMAX: {
    2822         606 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts,
    2823             :                      Depth + 1);
    2824         606 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2825             : 
    2826             :     // UMAX - we know that the result will have the maximum of the
    2827             :     // known one leading bits of the inputs.
    2828         303 :     unsigned LeadOne = Known.countMinLeadingOnes();
    2829         606 :     LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes());
    2830             : 
    2831         606 :     Known.Zero &= Known2.Zero;
    2832         606 :     Known.One &= Known2.One;
    2833         606 :     Known.One.setHighBits(LeadOne);
    2834             :     break;
    2835             :   }
    2836        2813 :   case ISD::SMIN:
    2837             :   case ISD::SMAX: {
    2838        5626 :     computeKnownBits(Op.getOperand(0), Known, DemandedElts,
    2839             :                      Depth + 1);
    2840             :     // If we don't know any bits, early out.
    2841        8439 :     if (!Known.One && !Known.Zero)
    2842             :       break;
    2843           0 :     computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
    2844           0 :     Known.Zero &= Known2.Zero;
    2845           0 :     Known.One &= Known2.One;
    2846             :     break;
    2847             :   }
    2848     1001319 :   case ISD::FrameIndex:
    2849             :   case ISD::TargetFrameIndex:
    2850     1001319 :     if (unsigned Align = InferPtrAlignment(Op)) {
    2851             :       // The low bits are known zero if the pointer is aligned.
    2852     2002638 :       Known.Zero.setLowBits(Log2_32(Align));
    2853             :       break;
    2854           0 :     }
    2855             :     break;
    2856             : 
    2857     6619174 :   default:
    2858     6619174 :     if (Opcode < ISD::BUILTIN_OP_END)
    2859             :       break;
    2860             :     LLVM_FALLTHROUGH;
    2861             :   case ISD::INTRINSIC_WO_CHAIN:
    2862             :   case ISD::INTRINSIC_W_CHAIN:
    2863             :   case ISD::INTRINSIC_VOID:
    2864             :     // Allow the target to implement this method for its nodes.
    2865     3108935 :     TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
    2866     3108935 :     break;
    2867             :   }
    2868             : 
    2869             :   assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?");
    2870             : }
    2871             : 
    2872       72949 : SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
    2873             :                                                              SDValue N1) const {
    2874             :   // X + 0 never overflow
    2875       72949 :   if (isNullConstant(N1))
    2876             :     return OFK_Never;
    2877             : 
    2878      145898 :   KnownBits N1Known;
    2879       72949 :   computeKnownBits(N1, N1Known);
    2880       72949 :   if (N1Known.Zero.getBoolValue()) {
    2881      142186 :     KnownBits N0Known;
    2882       71104 :     computeKnownBits(N0, N0Known);
    2883             : 
    2884             :     bool overflow;
    2885      639936 :     (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
    2886       71104 :     if (!overflow)
    2887          22 :       return OFK_Never;
    2888             :   }
    2889             : 
    2890             :   // mulhi + 1 never overflow
    2891      218904 :   if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
    2892      437808 :       (~N1Known.Zero & 0x01) == ~N1Known.Zero)
    2893             :     return OFK_Never;
    2894             : 
    2895      145772 :   if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
    2896          82 :     KnownBits N0Known;
    2897          41 :     computeKnownBits(N0, N0Known);
    2898             : 
    2899         451 :     if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
    2900           0 :       return OFK_Never;
    2901             :   }
    2902             : 
    2903             :   return OFK_Sometime;
    2904             : }
    2905             : 
    2906       36494 : bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
    2907       72988 :   EVT OpVT = Val.getValueType();
    2908       36494 :   unsigned BitWidth = OpVT.getScalarSizeInBits();
    2909             : 
    2910             :   // Is the constant a known power of 2?
    2911       34166 :   if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
    2912       68332 :     return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
    2913             : 
    2914             :   // A left-shift of a constant one will have exactly one bit set because
    2915             :   // shifting the bit off the end is undefined.
    2916        4656 :   if (Val.getOpcode() == ISD::SHL) {
    2917          94 :     auto *C = isConstOrConstSplat(Val.getOperand(0));
    2918          91 :     if (C && C->getAPIntValue() == 1)
    2919             :       return true;
    2920             :   }
    2921             : 
    2922             :   // Similarly, a logical right-shift of a constant sign-bit will have exactly
    2923             :   // one bit set.
    2924        4574 :   if (Val.getOpcode() == ISD::SRL) {
    2925          46 :     auto *C = isConstOrConstSplat(Val.getOperand(0));
    2926          33 :     if (C && C->getAPIntValue().isSignMask())
    2927             :       return true;
    2928             :   }
    2929             : 
    2930             :   // Are all operands of a build vector constant powers of two?
    2931        4564 :   if (Val.getOpcode() == ISD::BUILD_VECTOR)
    2932        6560 :     if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
    2933        1869 :           if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
    2934        3738 :             return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
    2935             :           return false;
    2936             :         }))
    2937             :       return true;
    2938             : 
    2939             :   // More could be done here, though the above checks are enough
    2940             :   // to handle some common cases.
    2941             : 
    2942             :   // Fall back to computeKnownBits to catch other known cases.
    2943        4236 :   KnownBits Known;
    2944        2118 :   computeKnownBits(Val, Known);
    2945        2193 :   return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
    2946             : }
    2947             : 
    2948      273459 : unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
    2949      546918 :   EVT VT = Op.getValueType();
    2950      273459 :   APInt DemandedElts = VT.isVector()
    2951             :                            ? APInt::getAllOnesValue(VT.getVectorNumElements())
    2952      546918 :                            : APInt(1, 1);
    2953      546918 :   return ComputeNumSignBits(Op, DemandedElts, Depth);
    2954             : }
    2955             : 
    2956      290659 : unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
    2957             :                                           unsigned Depth) const {
    2958      581318 :   EVT VT = Op.getValueType();
    2959             :   assert(VT.isInteger() && "Invalid VT!");
    2960      290659 :   unsigned VTBits = VT.getScalarSizeInBits();
    2961      290659 :   unsigned NumElts = DemandedElts.getBitWidth();
    2962             :   unsigned Tmp, Tmp2;
    2963      290659 :   unsigned FirstAnswer = 1;
    2964             : 
    2965      290659 :   if (Depth == 6)
    2966             :     return 1;  // Limit search depth.
    2967             : 
    2968      290449 :   if (!DemandedElts)
    2969             :     return 1;  // No demanded elts, better to assume we don't know anything.
    2970             : 
    2971      580898 :   switch (Op.getOpcode()) {
    2972      214793 :   default: break;
    2973        3162 :   case ISD::AssertSext:
    2974        9486 :     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
    2975       78818 :     return VTBits-Tmp+1;
    2976        7130 :   case ISD::AssertZext:
    2977       21390 :     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
    2978        7130 :     return VTBits-Tmp;
    2979             : 
    2980       19603 :   case ISD::Constant: {
    2981       39206 :     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
    2982       19603 :     return Val.getNumSignBits();
    2983             :   }
    2984             : 
    2985        3166 :   case ISD::BUILD_VECTOR:
    2986        3166 :     Tmp = VTBits;
    2987       13949 :     for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
    2988        7617 :       if (!DemandedElts[i])
    2989        3547 :         continue;
    2990             : 
    2991        8140 :       SDValue SrcOp = Op.getOperand(i);
    2992        8140 :       Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1);
    2993             : 
    2994             :       // BUILD_VECTOR can implicitly truncate sources, we must handle this.
    2995        4070 :       if (SrcOp.getValueSizeInBits() != VTBits) {
    2996             :         assert(SrcOp.getValueSizeInBits() > VTBits &&
    2997             :                "Expected BUILD_VECTOR implicit truncation");
    2998          66 :         unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
    2999          66 :         Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
    3000             :       }
    3001        4070 :       Tmp = std::min(Tmp, Tmp2);
    3002        3166 :     }
    3003        3166 :     return Tmp;
    3004             : 
    3005         650 :   case ISD::VECTOR_SHUFFLE: {
    3006             :     // Collect the minimum number of sign bits that are shared by every vector
    3007             :     // element referenced by the shuffle.
    3008        1300 :     APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
    3009         650 :     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
    3010             :     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
    3011        4807 :     for (unsigned i = 0; i != NumElts; ++i) {
    3012        8350 :       int M = SVN->getMaskElt(i);
    3013        4175 :       if (!DemandedElts[i])
    3014         223 :         continue;
    3015             :       // For UNDEF elements, we don't know anything about the common state of
    3016             :       // the shuffle result.
    3017        3952 :       if (M < 0)
    3018          36 :         return 1;
    3019        3934 :       if ((unsigned)M < NumElts)
    3020        2180 :         DemandedLHS.setBit((unsigned)M % NumElts);
    3021             :       else
    3022        1754 :         DemandedRHS.setBit((unsigned)M % NumElts);
    3023             :     }
    3024         632 :     Tmp = std::numeric_limits<unsigned>::max();
    3025         632 :     if (!!DemandedLHS)
    3026        1254 :       Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
    3027         632 :     if (!!DemandedRHS) {
    3028        1050 :       Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
    3029         525 :       Tmp = std::min(Tmp, Tmp2);
    3030             :     }
    3031             :     // If we don't know anything, early out and try computeKnownBits fall-back.
    3032         632 :     if (Tmp == 1)
    3033             :       break;
    3034             :     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
    3035             :     return Tmp;
    3036             :   }
    3037             : 
    3038         813 :   case ISD::SIGN_EXTEND:
    3039             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
    3040        1626 :     Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
    3041        1626 :     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
    3042             : 
    3043        2679 :   case ISD::SIGN_EXTEND_INREG:
    3044             :     // Max of the input and what this extends.
    3045       10716 :     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
    3046        2679 :     Tmp = VTBits-Tmp+1;
    3047             : 
    3048        5358 :     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3049        2679 :     return std::max(Tmp, Tmp2);
    3050             : 
    3051        1404 :   case ISD::SRA:
    3052        2808 :     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
    3053             :     // SRA X, C   -> adds C sign bits.
    3054        2808 :     if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1))) {
    3055        5328 :       APInt ShiftVal = C->getAPIntValue();
    3056        1332 :       ShiftVal += Tmp;
    3057        3930 :       Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
    3058             :     }
    3059        1404 :     return Tmp;
    3060        6754 :   case ISD::SHL:
    3061       13508 :     if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1))) {
    3062             :       // shl destroys sign bits.
    3063       12804 :       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3064       25608 :       if (C->getAPIntValue().uge(VTBits) ||      // Bad shift.
    3065       19206 :           C->getAPIntValue().uge(Tmp)) break;    // Shifted all sign bits out.
    3066         648 :       return Tmp - C->getZExtValue();
    3067         352 :     }
    3068             :     break;
    3069       11474 :   case ISD::AND:
    3070             :   case ISD::OR:
    3071             :   case ISD::XOR:    // NOT is handled here.
    3072             :     // Logical binary ops preserve the number of sign bits at the worst.
    3073       22948 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3074       11474 :     if (Tmp != 1) {
    3075       11250 :       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
    3076        5625 :       FirstAnswer = std::min(Tmp, Tmp2);
    3077             :       // We computed what we know about the sign bits as our first
    3078             :       // answer. Now proceed to the generic code that uses
    3079             :       // computeKnownBits, and pick whichever answer is better.
    3080             :     }
    3081             :     break;
    3082             : 
    3083         107 :   case ISD::SELECT:
    3084         214 :     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
    3085         107 :     if (Tmp == 1) return 1;  // Early out.
    3086         150 :     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
    3087          75 :     return std::min(Tmp, Tmp2);
    3088         157 :   case ISD::SELECT_CC:
    3089         314 :     Tmp = ComputeNumSignBits(Op.getOperand(2), Depth+1);
    3090         157 :     if (Tmp == 1) return 1;  // Early out.
    3091         214 :     Tmp2 = ComputeNumSignBits(Op.getOperand(3), Depth+1);
    3092         107 :     return std::min(Tmp, Tmp2);
    3093         105 :   case ISD::SMIN:
    3094             :   case ISD::SMAX:
    3095             :   case ISD::UMIN:
    3096             :   case ISD::UMAX:
    3097         210 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
    3098         105 :     if (Tmp == 1)
    3099             :       return 1;  // Early out.
    3100         190 :     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
    3101          95 :     return std::min(Tmp, Tmp2);
    3102          48 :   case ISD::SADDO:
    3103             :   case ISD::UADDO:
    3104             :   case ISD::SSUBO:
    3105             :   case ISD::USUBO:
    3106             :   case ISD::SMULO:
    3107             :   case ISD::UMULO:
    3108          48 :     if (Op.getResNo() != 1)
    3109             :       break;
    3110             :     // The boolean result conforms to getBooleanContents.  Fall through.
    3111             :     // If setcc returns 0/-1, all bits are sign bits.
    3112             :     // We know that we have an integer-based boolean since these operations
    3113             :     // are only available for integer.
    3114         192 :     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
    3115             :         TargetLowering::ZeroOrNegativeOneBooleanContent)
    3116             :       return VTBits;
    3117             :     break;
    3118        1452 :   case ISD::SETCC:
    3119             :     // If setcc returns 0/-1, all bits are sign bits.
    3120        4356 :     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
    3121             :         TargetLowering::ZeroOrNegativeOneBooleanContent)
    3122             :       return VTBits;
    3123             :     break;
    3124          52 :   case ISD::ROTL:
    3125             :   case ISD::ROTR:
    3126         156 :     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
    3127          52 :       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
    3128             : 
    3129             :       // Handle rotate right by N like a rotate left by 32-N.
    3130         104 :       if (Op.getOpcode() == ISD::ROTR)
    3131          28 :         RotAmt = (VTBits-RotAmt) & (VTBits-1);
    3132             : 
    3133             :       // If we aren't rotating out all of the known-in sign bits, return the
    3134             :       // number that are left.  This handles rotl(sext(x), 1) for example.
    3135         104 :       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3136          52 :       if (Tmp > RotAmt+1) return Tmp-RotAmt;
    3137             :     }
    3138             :     break;
    3139       15235 :   case ISD::ADD:
    3140             :   case ISD::ADDC:
    3141             :     // Add can have at most one carry bit.  Thus we know that the output
    3142             :     // is, at worst, one more bit than the inputs.
    3143       30470 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3144       15235 :     if (Tmp == 1) return 1;  // Early out.
    3145             : 
    3146             :     // Special case decrementing a value (ADD X, -1):
    3147        1976 :     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
    3148         404 :       if (CRHS->isAllOnesValue()) {
    3149          68 :         KnownBits Known;
    3150         114 :         computeKnownBits(Op.getOperand(0), Known, Depth+1);
    3151             : 
    3152             :         // If the input is known to be 0 or 1, the output is 0/-1, which is all
    3153             :         // sign bits set.
    3154         228 :         if ((Known.Zero | 1).isAllOnesValue())
    3155          46 :           return VTBits;
    3156             : 
    3157             :         // If we are subtracting one from a positive number, there is no carry
    3158             :         // out of the result.
    3159          49 :         if (Known.isNonNegative())
    3160             :           return Tmp;
    3161             :       }
    3162             : 
    3163        1480 :     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
    3164         740 :     if (Tmp2 == 1) return 1;
    3165         662 :     return std::min(Tmp, Tmp2)-1;
    3166             : 
    3167        3737 :   case ISD::SUB:
    3168        7474 :     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
    3169        3737 :     if (Tmp2 == 1) return 1;
    3170             : 
    3171             :     // Handle NEG.
    3172        1132 :     if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0)))
    3173         254 :       if (CLHS->isNullValue()) {
    3174          95 :         KnownBits Known;
    3175         140 :         computeKnownBits(Op.getOperand(1), Known, Depth+1);
    3176             :         // If the input is known to be 0 or 1, the output is 0/-1, which is all
    3177             :         // sign bits set.
    3178         280 :         if ((Known.Zero | 1).isAllOnesValue())
    3179          45 :           return VTBits;
    3180             : 
    3181             :         // If the input is known to be positive (the sign bit is known clear),
    3182             :         // the output of the NEG has the same number of sign bits as the input.
    3183          65 :         if (Known.isNonNegative())
    3184             :           return Tmp2;
    3185             : 
    3186             :         // Otherwise, we treat this like a SUB.
    3187             :       }
    3188             : 
    3189             :     // Sub can have at most one carry bit.  Thus we know that the output
    3190             :     // is, at worst, one more bit than the inputs.
    3191        1042 :     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3192         521 :     if (Tmp == 1) return 1;  // Early out.
    3193         422 :     return std::min(Tmp, Tmp2)-1;
    3194       25404 :   case ISD::TRUNCATE: {
    3195             :     // Check if the sign bits of source go down as far as the truncated value.
    3196       50808 :     unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
    3197       50808 :     unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
    3198       25404 :     if (NumSrcSignBits > (NumSrcBits - VTBits))
    3199        2082 :       return NumSrcSignBits - (NumSrcBits - VTBits);
    3200             :     break;
    3201             :   }
    3202          24 :   case ISD::EXTRACT_ELEMENT: {
    3203          48 :     const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
    3204          24 :     const int BitWidth = Op.getValueSizeInBits();
    3205          48 :     const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
    3206             : 
    3207             :     // Get reverse index (starting from 1), Op1 value indexes elements from
    3208             :     // little end. Sign starts at big end.
    3209          48 :     const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
    3210             : 
    3211             :     // If the sign portion ends in our element the subtraction gives correct
    3212             :     // result. Otherwise it gives either negative or > bitwidth result
    3213          72 :     return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
    3214             :   }
    3215         104 :   case ISD::INSERT_VECTOR_ELT: {
    3216         208 :     SDValue InVec = Op.getOperand(0);
    3217         208 :     SDValue InVal = Op.getOperand(1);
    3218         208 :     SDValue EltNo = Op.getOperand(2);
    3219         208 :     unsigned NumElts = InVec.getValueType().getVectorNumElements();
    3220             : 
    3221         100 :     ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
    3222         100 :     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
    3223             :       // If we know the element index, split the demand between the
    3224             :       // source vector and the inserted element.
    3225         100 :       unsigned EltIdx = CEltNo->getZExtValue();
    3226             : 
    3227             :       // If we demand the inserted element then get its sign bits.
    3228         100 :       Tmp = std::numeric_limits<unsigned>::max();
    3229         100 :       if (DemandedElts[EltIdx]) {
    3230             :         // TODO - handle implicit truncation of inserted elements.
    3231          98 :         if (InVal.getScalarValueSizeInBits() != VTBits)
    3232             :           break;
    3233          98 :         Tmp = ComputeNumSignBits(InVal, Depth + 1);
    3234             :       }
    3235             : 
    3236             :       // If we demand the source vector then get its sign bits, and determine
    3237             :       // the minimum.
    3238         200 :       APInt VectorElts = DemandedElts;
    3239         100 :       VectorElts.clearBit(EltIdx);
    3240         100 :       if (!!VectorElts) {
    3241          65 :         Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1);
    3242          65 :         Tmp = std::min(Tmp, Tmp2);
    3243             :       }
    3244             :     } else {
    3245             :       // Unknown element index, so ignore DemandedElts and demand them all.
    3246           4 :       Tmp = ComputeNumSignBits(InVec, Depth + 1);
    3247           4 :       Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
    3248           4 :       Tmp = std::min(Tmp, Tmp2);
    3249             :     }
    3250             :     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
    3251         104 :     return Tmp;
    3252             :   }
    3253       13807 :   case ISD::EXTRACT_VECTOR_ELT: {
    3254       27614 :     SDValue InVec = Op.getOperand(0);
    3255       27614 :     SDValue EltNo = Op.getOperand(1);
    3256       27614 :     EVT VecVT = InVec.getValueType();
    3257       13807 :     const unsigned BitWidth = Op.getValueSizeInBits();
    3258       27614 :     const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
    3259       13807 :     const unsigned NumSrcElts = VecVT.getVectorNumElements();
    3260             : 
    3261             :     // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
    3262             :     // anything about sign bits. But if the sizes match we can derive knowledge
    3263             :     // about sign bits from the vector operand.
    3264       13807 :     if (BitWidth != EltBitWidth)
    3265             :       break;
    3266             : 
    3267             :     // If we know the element index, just demand that vector element, else for
    3268             :     // an unknown element index, ignore DemandedElts and demand them all.
    3269       25684 :     APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
    3270       12842 :     ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
    3271       12842 :     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
    3272       12842 :       DemandedSrcElts =
    3273       25684 :           APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
    3274             : 
    3275       12842 :     return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
    3276             :   }
    3277        1022 :   case ISD::EXTRACT_SUBVECTOR: {
    3278             :     // If we know the element index, just demand that subvector elements,
    3279             :     // otherwise demand them all.
    3280        2044 :     SDValue Src = Op.getOperand(0);
    3281        3066 :     ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
    3282        2044 :     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
    3283        3066 :     if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
    3284             :       // Offset the demanded elts by the subvector index.
    3285        1022 :       uint64_t Idx = SubIdx->getZExtValue();
    3286        3066 :       APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx);
    3287        1022 :       return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
    3288             :     }
    3289           0 :     return ComputeNumSignBits(Src, Depth + 1);
    3290             :   }
    3291         691 :   case ISD::CONCAT_VECTORS:
    3292             :     // Determine the minimum number of sign bits across all demanded
    3293             :     // elts of the input vectors. Early out if the result is already 1.
    3294         691 :     Tmp = std::numeric_limits<unsigned>::max();
    3295        2073 :     EVT SubVectorVT = Op.getOperand(0).getValueType();
    3296         691 :     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
    3297        1382 :     unsigned NumSubVectors = Op.getNumOperands();
    3298        1543 :     for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
    3299        1567 :       APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
    3300        2556 :       DemandedSub = DemandedSub.trunc(NumSubVectorElts);
    3301         852 :       if (!DemandedSub)
    3302         137 :         continue;
    3303        1430 :       Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
    3304         715 :       Tmp = std::min(Tmp, Tmp2);
    3305         691 :     }
    3306             :     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
    3307         691 :     return Tmp;
    3308             :   }
    3309             : 
    3310             :   // If we are looking at the loaded value of the SDNode.
    3311      214793 :   if (Op.getResNo() == 0) {
    3312             :     // Handle LOADX separately here. EXTLOAD case will fallthrough.
    3313       44687 :     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
    3314       44687 :       unsigned ExtType = LD->getExtensionType();
    3315       44687 :       switch (ExtType) {
    3316             :         default: break;
    3317        2335 :         case ISD::SEXTLOAD:    // '17' bits known
    3318        4670 :           Tmp = LD->getMemoryVT().getScalarSizeInBits();
    3319        2335 :           return VTBits-Tmp+1;
    3320        1408 :         case ISD::ZEXTLOAD:    // '16' bits known
    3321        2816 :           Tmp = LD->getMemoryVT().getScalarSizeInBits();
    3322        1408 :           return VTBits-Tmp;
    3323             :       }
    3324             :     }
    3325             :   }
    3326             : 
    3327             :   // Allow the target to implement this method for its nodes.
    3328      413560 :   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
    3329      404783 :       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
    3330      614930 :       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
    3331      201607 :       Op.getOpcode() == ISD::INTRINSIC_VOID) {
    3332             :     unsigned NumBits =
    3333        9443 :         TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
    3334        9443 :     if (NumBits > 1)
    3335        2138 :       FirstAnswer = std::max(FirstAnswer, NumBits);
    3336             :   }
    3337             : 
    3338             :   // Finally, if we can prove that the top bits of the result are 0's or 1's,
    3339             :   // use this information.
    3340      422100 :   KnownBits Known;
    3341      211050 :   computeKnownBits(Op, Known, DemandedElts, Depth);
    3342             : 
    3343      422100 :   APInt Mask;
    3344      211050 :   if (Known.isNonNegative()) {        // sign bit is 0
    3345       39322 :     Mask = Known.Zero;
    3346      171728 :   } else if (Known.isNegative()) {  // sign bit is 1;
    3347          47 :     Mask = Known.One;
    3348             :   } else {
    3349             :     // Nothing known.
    3350      171681 :     return FirstAnswer;
    3351             :   }
    3352             : 
    3353             :   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
    3354             :   // the number of identical bits in the top of the input value.
    3355      196845 :   Mask = ~Mask;
    3356       39369 :   Mask <<= Mask.getBitWidth()-VTBits;
    3357             :   // Return # leading zeros.  We use 'min' here in case Val was zero before
    3358             :   // shifting.  We don't want to return '64' as for an i32 "0".
    3359      118107 :   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
    3360             : }
    3361             : 
    3362     2780408 : bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
    3363     6546754 :   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
    3364     4008942 :       !isa<ConstantSDNode>(Op.getOperand(1)))
    3365             :     return false;
    3366             : 
    3367     3757894 :   if (Op.getOpcode() == ISD::OR &&
    3368     1031800 :       !MaskedValueIsZero(Op.getOperand(0),
    3369             :                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
    3370             :     return false;
    3371             : 
    3372             :   return true;
    3373             : }
    3374             : 
    3375         593 : bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
    3376             :   // If we're told that NaNs won't happen, assume they won't.
    3377         593 :   if (getTarget().Options.NoNaNsFPMath)
    3378             :     return true;
    3379             : 
    3380         798 :   if (Op->getFlags().hasNoNaNs())
    3381             :     return true;
    3382             : 
    3383             :   // If the value is a constant, we can obviously see if it is a NaN or not.
    3384          43 :   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
    3385          86 :     return !C->getValueAPF().isNaN();
    3386             : 
    3387             :   // TODO: Recognize more cases here.
    3388             : 
    3389             :   return false;
    3390             : }
    3391             : 
    3392         172 : bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
    3393             :   // If the value is a constant, we can obviously see if it is a zero or not.
    3394          26 :   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
    3395          26 :     return !C->isZero();
    3396             : 
    3397             :   // TODO: Recognize more cases here.
    3398         292 :   switch (Op.getOpcode()) {
    3399             :   default: break;
    3400           0 :   case ISD::OR:
    3401           0 :     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
    3402           0 :       return !C->isNullValue();
    3403             :     break;
    3404             :   }
    3405             : 
    3406             :   return false;
    3407             : }
    3408             : 
    3409        7394 : bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
    3410             :   // Check the obvious case.
    3411        5367 :   if (A == B) return true;
    3412             : 
    3413             :   // For for negative and positive zero.
    3414         541 :   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
    3415          12 :     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
    3416          16 :       if (CA->isZero() && CB->isZero()) return true;
    3417             : 
    3418             :   // Otherwise they may not be equal.
    3419             :   return false;
    3420             : }
    3421             : 
    3422     1707483 : bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
    3423             :   assert(A.getValueType() == B.getValueType() &&
    3424             :          "Values must have the same type");
    3425     5122449 :   KnownBits AKnown, BKnown;
    3426     1707483 :   computeKnownBits(A, AKnown);
    3427     1707483 :   computeKnownBits(B, BKnown);
    3428    10244898 :   return (AKnown.Zero | BKnown.Zero).isAllOnesValue();
    3429             : }
    3430             : 
    3431       38711 : static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
    3432             :                                   ArrayRef<SDValue> Ops,
    3433             :                                   SelectionDAG &DAG) {
    3434             :   assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
    3435             :   assert(llvm::all_of(Ops,
    3436             :                       [Ops](SDValue Op) {
    3437             :                         return Ops[0].getValueType() == Op.getValueType();
    3438             :                       }) &&
    3439             :          "Concatenation of vectors with inconsistent value types!");
    3440             :   assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==
    3441             :              VT.getVectorNumElements() &&
    3442             :          "Incorrect element count in vector concatenation!");
    3443             : 
    3444       38711 :   if (Ops.size() == 1)
    3445           0 :     return Ops[0];
    3446             : 
    3447             :   // Concat of UNDEFs is UNDEF.
    3448       77422 :   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
    3449         148 :     return DAG.getUNDEF(VT);
    3450             : 
    3451             :   // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
    3452             :   // simplified to one big BUILD_VECTOR.
    3453             :   // FIXME: Add support for SCALAR_TO_VECTOR as well.
    3454       38563 :   EVT SVT = VT.getScalarType();
    3455       38563 :   SmallVector<SDValue, 16> Elts;
    3456       40984 :   for (SDValue Op : Ops) {
    3457       80834 :     EVT OpVT = Op.getValueType();
    3458       80834 :     if (Op.isUndef())
    3459         505 :       Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
    3460       79824 :     else if (Op.getOpcode() == ISD::BUILD_VECTOR)
    3461        3832 :       Elts.append(Op->op_begin(), Op->op_end());
    3462             :     else
    3463       37996 :       return SDValue();
    3464             :   }
    3465             : 
    3466             :   // BUILD_VECTOR requires all inputs to be of the same type, find the
    3467             :   // maximum type and extend them all.
    3468       12127 :   for (SDValue Op : Elts)
    3469       20997 :     SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
    3470             : 
    3471         567 :   if (SVT.bitsGT(VT.getScalarType()))
    3472        4879 :     for (SDValue &Op : Elts)
    3473       13332 :       Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
    3474        8888 :                ? DAG.getZExtOrTrunc(Op, DL, SVT)
    3475             :                : DAG.getSExtOrTrunc(Op, DL, SVT);
    3476             : 
    3477         567 :   SDValue V = DAG.getBuildVector(VT, DL, Elts);
    3478         567 :   NewSDValueDbgMsg(V, "New node fold concat vectors: ");
    3479         567 :   return V;
    3480             : }
    3481             : 
    3482             : /// Gets or creates the specified node.
    3483     2369407 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
    3484     4738814 :   FoldingSetNodeID ID;
    3485     2369407 :   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
    3486     2369407 :   void *IP = nullptr;
    3487     2369407 :   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    3488     2023712 :     return SDValue(E, 0);
    3489             : 
    3490      345695 :   auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
    3491      691390 :                               getVTList(VT));
    3492      691390 :   CSEMap.InsertNode(N, IP);
    3493             : 
    3494      345695 :   InsertNode(N);
    3495      345695 :   SDValue V = SDValue(N, 0);
    3496      345695 :   NewSDValueDbgMsg(V, "Creating new node: ");
    3497      345695 :   return V;
    3498             : }
    3499             : 
    3500     3517460 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    3501             :                               SDValue Operand, const SDNodeFlags Flags) {
    3502             :   // Constant fold unary operations with an integer constant operand. Even
    3503             :   // opaque constant will be folded, because the folding of unary operations
    3504             :   // doesn't create new constants with different values. Nevertheless, the
    3505             :   // opaque flag is preserved during folding to prevent future folding with
    3506             :   // other constants.
    3507      288787 :   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
    3508      288787 :     const APInt &Val = C->getAPIntValue();
    3509      288787 :     switch (Opcode) {
    3510             :     default: break;
    3511       28954 :     case ISD::SIGN_EXTEND:
    3512       57908 :       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
    3513       86862 :                          C->isTargetOpcode(), C->isOpaque());
    3514      195675 :     case ISD::ANY_EXTEND:
    3515             :     case ISD::ZERO_EXTEND:
    3516             :     case ISD::TRUNCATE:
    3517      391350 :       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
    3518      587025 :                          C->isTargetOpcode(), C->isOpaque());
    3519         255 :     case ISD::UINT_TO_FP:
    3520             :     case ISD::SINT_TO_FP: {
    3521             :       APFloat apf(EVTToAPFloatSemantics(VT),
    3522        1020 :                   APInt::getNullValue(VT.getSizeInBits()));
    3523         255 :       (void)apf.convertFromAPInt(Val,
    3524             :                                  Opcode==ISD::SINT_TO_FP,
    3525             :                                  APFloat::rmNearestTiesToEven);
    3526         255 :       return getConstantFP(apf, DL, VT);
    3527             :     }
    3528        6786 :     case ISD::BITCAST:
    3529        6800 :       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
    3530          21 :         return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
    3531        7417 :       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
    3532         957 :         return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
    3533        6674 :       if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
    3534         321 :         return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
    3535        6353 :       if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
    3536           0 :         return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
    3537             :       break;
    3538          14 :     case ISD::ABS:
    3539          42 :       return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
    3540          42 :                          C->isOpaque());
    3541         403 :     case ISD::BITREVERSE:
    3542        1209 :       return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
    3543        1209 :                          C->isOpaque());
    3544         170 :     case ISD::BSWAP:
    3545         510 :       return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
    3546         510 :                          C->isOpaque());
    3547         390 :     case ISD::CTPOP:
    3548         780 :       return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
    3549        1170 :                          C->isOpaque());
    3550       27639 :     case ISD::CTLZ:
    3551             :     case ISD::CTLZ_ZERO_UNDEF:
    3552       55278 :       return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
    3553       82917 :                          C->isOpaque());
    3554        1320 :     case ISD::CTTZ:
    3555             :     case ISD::CTTZ_ZERO_UNDEF:
    3556        2640 :       return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
    3557        3960 :                          C->isOpaque());
    3558         265 :     case ISD::FP16_TO_FP: {
    3559             :       bool Ignored;
    3560             :       APFloat FPV(APFloat::IEEEhalf(),
    3561        1060 :                   (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
    3562             : 
    3563             :       // This can return overflow, underflow, or inexact; we don't care.
    3564             :       // FIXME need to be more flexible about rounding mode.
    3565         265 :       (void)FPV.convert(EVTToAPFloatSemantics(VT),
    3566             :                         APFloat::rmNearestTiesToEven, &Ignored);
    3567         265 :       return getConstantFP(FPV, DL, VT);
    3568             :     }
    3569             :     }
    3570             :   }
    3571             : 
    3572             :   // Constant fold unary operations with a floating point constant operand.
    3573        4734 :   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
    3574       10462 :     APFloat V = C->getValueAPF();    // make copy
    3575        4734 :     switch (Opcode) {
    3576         196 :     case ISD::FNEG:
    3577         196 :       V.changeSign();
    3578        3936 :       return getConstantFP(V, DL, VT);
    3579           2 :     case ISD::FABS:
    3580           2 :       V.clearSign();
    3581           2 :       return getConstantFP(V, DL, VT);
    3582          16 :     case ISD::FCEIL: {
    3583          16 :       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
    3584          16 :       if (fs == APFloat::opOK || fs == APFloat::opInexact)
    3585          16 :         return getConstantFP(V, DL, VT);
    3586             :       break;
    3587             :     }
    3588          13 :     case ISD::FTRUNC: {
    3589          13 :       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
    3590          13 :       if (fs == APFloat::opOK || fs == APFloat::opInexact)
    3591          13 :         return getConstantFP(V, DL, VT);
    3592             :       break;
    3593             :     }
    3594          13 :     case ISD::FFLOOR: {
    3595          13 :       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
    3596          13 :       if (fs == APFloat::opOK || fs == APFloat::opInexact)
    3597          13 :         return getConstantFP(V, DL, VT);
    3598             :       break;
    3599             :     }
    3600         103 :     case ISD::FP_EXTEND: {
    3601             :       bool ignored;
    3602             :       // This can return overflow, underflow, or inexact; we don't care.
    3603             :       // FIXME need to be more flexible about rounding mode.
    3604         103 :       (void)V.convert(EVTToAPFloatSemantics(VT),
    3605             :                       APFloat::rmNearestTiesToEven, &ignored);
    3606         103 :       return getConstantFP(V, DL, VT);
    3607             :     }
    3608         625 :     case ISD::FP_TO_SINT:
    3609             :     case ISD::FP_TO_UINT: {
    3610             :       bool ignored;
    3611         625 :       APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
    3612             :       // FIXME need to be more flexible about rounding mode.
    3613             :       APFloat::opStatus s =
    3614         625 :           V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
    3615         625 :       if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
    3616             :         break;
    3617        1194 :       return getConstant(IntVal, DL, VT);
    3618             :     }
    3619        2860 :     case ISD::BITCAST:
    3620        3246 :       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
    3621         579 :         return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
    3622        7317 :       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
    3623        6975 :         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
    3624         870 :       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
    3625         792 :         return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
    3626             :       break;
    3627          18 :     case ISD::FP_TO_FP16: {
    3628             :       bool Ignored;
    3629             :       // This can return overflow, underflow, or inexact; we don't care.
    3630             :       // FIXME need to be more flexible about rounding mode.
    3631          18 :       (void)V.convert(APFloat::IEEEhalf(),
    3632             :                       APFloat::rmNearestTiesToEven, &Ignored);
    3633          36 :       return getConstant(V.bitcastToAPInt(), DL, VT);
    3634             :     }
    3635             :     }
    3636             :   }
    3637             : 
    3638             :   // Constant fold unary operations with a vector integer or float operand.
    3639      156282 :   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
    3640      156282 :     if (BV->isConstant()) {
    3641             :       switch (Opcode) {
    3642             :       default:
    3643             :         // FIXME: Entirely reasonable to perform folding of other unary
    3644             :         // operations here as the need arises.
    3645             :         break;
    3646        1608 :       case ISD::FNEG:
    3647             :       case ISD::FABS:
    3648             :       case ISD::FCEIL:
    3649             :       case ISD::FTRUNC:
    3650             :       case ISD::FFLOOR:
    3651             :       case ISD::FP_EXTEND:
    3652             :       case ISD::FP_TO_SINT:
    3653             :       case ISD::FP_TO_UINT:
    3654             :       case ISD::TRUNCATE:
    3655             :       case ISD::UINT_TO_FP:
    3656             :       case ISD::SINT_TO_FP:
    3657             :       case ISD::ABS:
    3658             :       case ISD::BITREVERSE:
    3659             :       case ISD::BSWAP:
    3660             :       case ISD::CTLZ:
    3661             :       case ISD::CTLZ_ZERO_UNDEF:
    3662             :       case ISD::CTTZ:
    3663             :       case ISD::CTTZ_ZERO_UNDEF:
    3664             :       case ISD::CTPOP: {
    3665        1608 :         SDValue Ops = { Operand };
    3666        3216 :         if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
    3667        1588 :           return Fold;
    3668             :       }
    3669             :       }
    3670             :     }
    3671             :   }
    3672             : 
    3673     6513228 :   unsigned OpOpcode = Operand.getNode()->getOpcode();
    3674     3256614 :   switch (Opcode) {
    3675     1128695 :   case ISD::TokenFactor:
    3676             :   case ISD::MERGE_VALUES:
    3677             :   case ISD::CONCAT_VECTORS:
    3678     1128695 :     return Operand;         // Factor, merge or concat of one node?  No need.
    3679           0 :   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
    3680        7896 :   case ISD::FP_EXTEND:
    3681             :     assert(VT.isFloatingPoint() &&
    3682             :            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
    3683       15806 :     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
    3684             :     assert((!VT.isVector() ||
    3685             :             VT.getVectorNumElements() ==
    3686             :             Operand.getValueType().getVectorNumElements()) &&
    3687             :            "Vector element count mismatch!");
    3688             :     assert(Operand.getValueType().bitsLT(VT) &&
    3689             :            "Invalid fpext node, dst < src!");
    3690       11202 :     if (Operand.isUndef())
    3691           8 :       return getUNDEF(VT);
    3692             :     break;
    3693       51120 :   case ISD::SIGN_EXTEND:
    3694             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3695             :            "Invalid SIGN_EXTEND!");
    3696      102312 :     if (Operand.getValueType() == VT) return Operand;   // noop extension
    3697             :     assert((!VT.isVector() ||
    3698             :             VT.getVectorNumElements() ==
    3699             :             Operand.getValueType().getVectorNumElements()) &&
    3700             :            "Vector element count mismatch!");
    3701             :     assert(Operand.getValueType().bitsLT(VT) &&
    3702             :            "Invalid sext node, dst < src!");
    3703       48539 :     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
    3704         532 :       return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
    3705       48273 :     else if (OpOpcode == ISD::UNDEF)
    3706             :       // sext(undef) = 0, because the top bits will all be the same.
    3707          55 :       return getConstant(0, DL, VT);
    3708             :     break;
    3709      152551 :   case ISD::ZERO_EXTEND:
    3710             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3711             :            "Invalid ZERO_EXTEND!");
    3712      306698 :     if (Operand.getValueType() == VT) return Operand;   // noop extension
    3713             :     assert((!VT.isVector() ||
    3714             :             VT.getVectorNumElements() ==
    3715             :             Operand.getValueType().getVectorNumElements()) &&
    3716             :            "Vector element count mismatch!");
    3717             :     assert(Operand.getValueType().bitsLT(VT) &&
    3718             :            "Invalid zext node, dst < src!");
    3719      147360 :     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
    3720         706 :       return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
    3721      147007 :     else if (OpOpcode == ISD::UNDEF)
    3722             :       // zext(undef) = 0, because the top bits will be zero.
    3723         809 :       return getConstant(0, DL, VT);
    3724             :     break;
    3725      167725 :   case ISD::ANY_EXTEND:
    3726             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3727             :            "Invalid ANY_EXTEND!");
    3728      336963 :     if (Operand.getValueType() == VT) return Operand;   // noop extension
    3729             :     assert((!VT.isVector() ||
    3730             :             VT.getVectorNumElements() ==
    3731             :             Operand.getValueType().getVectorNumElements()) &&
    3732             :            "Vector element count mismatch!");
    3733             :     assert(Operand.getValueType().bitsLT(VT) &&
    3734             :            "Invalid anyext node, dst < src!");
    3735             : 
    3736       98034 :     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
    3737             :         OpOpcode == ISD::ANY_EXTEND)
    3738             :       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
    3739        3276 :       return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
    3740       96396 :     else if (OpOpcode == ISD::UNDEF)
    3741         593 :       return getUNDEF(VT);
    3742             : 
    3743             :     // (ext (trunx x)) -> x
    3744       95803 :     if (OpOpcode == ISD::TRUNCATE) {
    3745       22436 :       SDValue OpOp = Operand.getOperand(0);
    3746       22440 :       if (OpOp.getValueType() == VT)
    3747        8819 :         return OpOp;
    3748             :     }
    3749             :     break;
    3750      576860 :   case ISD::TRUNCATE:
    3751             :     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
    3752             :            "Invalid TRUNCATE!");
    3753     1158280 :     if (Operand.getValueType() == VT) return Operand;   // noop truncate
    3754             :     assert((!VT.isVector() ||
    3755             :             VT.getVectorNumElements() ==
    3756             :             Operand.getValueType().getVectorNumElements()) &&
    3757             :            "Vector element count mismatch!");
    3758             :     assert(Operand.getValueType().bitsGT(VT) &&
    3759             :            "Invalid truncate node, src < dst!");
    3760      322184 :     if (OpOpcode == ISD::TRUNCATE)
    3761       67936 :       return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
    3762      288216 :     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
    3763             :         OpOpcode == ISD::ANY_EXTEND) {
    3764             :       // If the source is smaller than the dest, we still need an extend.
    3765       40296 :       if (Operand.getOperand(0).getValueType().getScalarType()
    3766       10074 :             .bitsLT(VT.getScalarType()))
    3767        8408 :         return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
    3768       17610 :       if (Operand.getOperand(0).getValueType().bitsGT(VT))
    3769         480 :         return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
    3770       11260 :       return Operand.getOperand(0);
    3771             :     }
    3772      278142 :     if (OpOpcode == ISD::UNDEF)
    3773        9551 :       return getUNDEF(VT);
    3774             :     break;
    3775         373 :   case ISD::ABS:
    3776             :     assert(VT.isInteger() && VT == Operand.getValueType() &&
    3777             :            "Invalid ABS!");
    3778         373 :     if (OpOpcode == ISD::UNDEF)
    3779           1 :       return getUNDEF(VT);
    3780             :     break;
    3781         877 :   case ISD::BSWAP:
    3782             :     assert(VT.isInteger() && VT == Operand.getValueType() &&
    3783             :            "Invalid BSWAP!");
    3784             :     assert((VT.getScalarSizeInBits() % 16 == 0) &&
    3785             :            "BSWAP types must be a multiple of 16 bits!");
    3786         877 :     if (OpOpcode == ISD::UNDEF)
    3787           4 :       return getUNDEF(VT);
    3788             :     break;
    3789         552 :   case ISD::BITREVERSE:
    3790             :     assert(VT.isInteger() && VT == Operand.getValueType() &&
    3791             :            "Invalid BITREVERSE!");
    3792         552 :     if (OpOpcode == ISD::UNDEF)
    3793           4 :       return getUNDEF(VT);
    3794             :     break;
    3795      614591 :   case ISD::BITCAST:
    3796             :     // Basic sanity checking.
    3797             :     assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
    3798             :            "Cannot BITCAST between types of different sizes!");
    3799     1233358 :     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
    3800      464397 :     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
    3801       58478 :       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
    3802      435158 :     if (OpOpcode == ISD::UNDEF)
    3803        5155 :       return getUNDEF(VT);
    3804             :     break;
    3805       19200 :   case ISD::SCALAR_TO_VECTOR:
    3806             :     assert(VT.isVector() && !Operand.getValueType().isVector() &&
    3807             :            (VT.getVectorElementType() == Operand.getValueType() ||
    3808             :             (VT.getVectorElementType().isInteger() &&
    3809             :              Operand.getValueType().isInteger() &&
    3810             :              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
    3811             :            "Illegal SCALAR_TO_VECTOR node!");
    3812       19200 :     if (OpOpcode == ISD::UNDEF)
    3813           1 :       return getUNDEF(VT);
    3814             :     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
    3815             :     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
    3816        3987 :         isa<ConstantSDNode>(Operand.getOperand(1)) &&
    3817       21641 :         Operand.getConstantOperandVal(1) == 0 &&
    3818        2589 :         Operand.getOperand(0).getValueType() == VT)
    3819        1386 :       return Operand.getOperand(0);
    3820             :     break;
    3821        3998 :   case ISD::FNEG:
    3822             :     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
    3823        3998 :     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
    3824             :       // FIXME: FNEG has no fast-math-flags to propagate; use the FSUB's flags?
    3825         118 :       return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1),
    3826         236 :                      Operand.getOperand(0), Operand.getNode()->getFlags());
    3827        3939 :     if (OpOpcode == ISD::FNEG)  // --X -> X
    3828           0 :       return Operand.getOperand(0);
    3829             :     break;
    3830        1322 :   case ISD::FABS:
    3831        1322 :     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
    3832           0 :       return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
    3833             :     break;
    3834             :   }
    3835             : 
    3836             :   SDNode *N;
    3837     1542001 :   SDVTList VTs = getVTList(VT);
    3838     1542001 :   SDValue Ops[] = {Operand};
    3839     1542001 :   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
    3840     3018275 :     FoldingSetNodeID ID;
    3841     1541690 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    3842     1541690 :     void *IP = nullptr;
    3843     1541690 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
    3844       65105 :       E->intersectFlagsWith(Flags);
    3845       65105 :       return SDValue(E, 0);
    3846             :     }
    3847             : 
    3848     2953170 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    3849     2953170 :     N->setFlags(Flags);
    3850     1476585 :     createOperands(N, Ops);
    3851     2953170 :     CSEMap.InsertNode(N, IP);
    3852             :   } else {
    3853         622 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    3854         311 :     createOperands(N, Ops);
    3855             :   }
    3856             : 
    3857     1476896 :   InsertNode(N);
    3858     1476896 :   SDValue V = SDValue(N, 0);
    3859     1476896 :   NewSDValueDbgMsg(V, "Creating new node: ");
    3860     1476896 :   return V;
    3861             : }
    3862             : 
    3863      440018 : static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
    3864             :                                         const APInt &C2) {
    3865      440018 :   switch (Opcode) {
    3866     1283586 :   case ISD::ADD:  return std::make_pair(C1 + C2, true);
    3867      161196 :   case ISD::SUB:  return std::make_pair(C1 - C2, true);
    3868       12579 :   case ISD::MUL:  return std::make_pair(C1 * C2, true);
    3869       58602 :   case ISD::AND:  return std::make_pair(C1 & C2, true);
    3870        8796 :   case ISD::OR:   return std::make_pair(C1 | C2, true);
    3871       13602 :   case ISD::XOR:  return std::make_pair(C1 ^ C2, true);
    3872      248336 :   case ISD::SHL:  return std::make_pair(C1 << C2, true);
    3873       40059 :   case ISD::SRL:  return std::make_pair(C1.lshr(C2), true);
    3874        2415 :   case ISD::SRA:  return std::make_pair(C1.ashr(C2), true);
    3875          48 :   case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
    3876           0 :   case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
    3877        2016 :   case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true);
    3878        2028 :   case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
    3879        1968 :   case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
    3880        1971 :   case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
    3881          28 :   case ISD::UDIV:
    3882          28 :     if (!C2.getBoolValue())
    3883             :       break;
    3884          84 :     return std::make_pair(C1.udiv(C2), true);
    3885          32 :   case ISD::UREM:
    3886          32 :     if (!C2.getBoolValue())
    3887             :       break;
    3888          96 :     return std::make_pair(C1.urem(C2), true);
    3889          99 :   case ISD::SDIV:
    3890          99 :     if (!C2.getBoolValue())
    3891             :       break;
    3892         297 :     return std::make_pair(C1.sdiv(C2), true);
    3893          57 :   case ISD::SREM:
    3894          57 :     if (!C2.getBoolValue())
    3895             :       break;
    3896         171 :     return std::make_pair(C1.srem(C2), true);
    3897             :   }
    3898      409572 :   return std::make_pair(APInt(1, 0), false);
    3899             : }
    3900             : 
    3901      440110 : SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
    3902             :                                              EVT VT, const ConstantSDNode *Cst1,
    3903             :                                              const ConstantSDNode *Cst2) {
    3904     1320239 :   if (Cst1->isOpaque() || Cst2->isOpaque())
    3905          92 :     return SDValue();
    3906             : 
    3907             :   std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
    3908     1320054 :                                             Cst2->getAPIntValue());
    3909      440018 :   if (!Folded.second)
    3910      102393 :     return SDValue();
    3911      337625 :   return getConstant(Folded.first, DL, VT);
    3912             : }
    3913             : 
    3914      254917 : SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
    3915             :                                        const GlobalAddressSDNode *GA,
    3916             :                                        const SDNode *N2) {
    3917      254917 :   if (GA->getOpcode() != ISD::GlobalAddress)
    3918          11 :     return SDValue();
    3919      254906 :   if (!TLI->isOffsetFoldingLegal(GA))
    3920      249705 :     return SDValue();
    3921        3579 :   const ConstantSDNode *Cst2 = dyn_cast<ConstantSDNode>(N2);
    3922             :   if (!Cst2)
    3923        1622 :     return SDValue();
    3924        3579 :   int64_t Offset = Cst2->getSExtValue();
    3925        3579 :   switch (Opcode) {
    3926             :   case ISD::ADD: break;
    3927           1 :   case ISD::SUB: Offset = -uint64_t(Offset); break;
    3928           8 :   default: return SDValue();
    3929             :   }
    3930        7142 :   return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT,
    3931        7142 :                           GA->getOffset() + uint64_t(Offset));
    3932             : }
    3933             : 
    3934     4932563 : bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
    3935     4932563 :   switch (Opcode) {
    3936       14151 :   case ISD::SDIV:
    3937             :   case ISD::UDIV:
    3938             :   case ISD::SREM:
    3939             :   case ISD::UREM: {
    3940             :     // If a divisor is zero/undef or any element of a divisor vector is
    3941             :     // zero/undef, the whole op is undef.
    3942             :     assert(Ops.size() == 2 && "Div/rem should have 2 operands");
    3943       14151 :     SDValue Divisor = Ops[1];
    3944       28302 :     if (Divisor.isUndef() || isNullConstant(Divisor))
    3945             :       return true;
    3946             : 
    3947       15112 :     return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
    3948        2994 :            llvm::any_of(Divisor->op_values(),
    3949       33168 :                         [](SDValue V) { return V.isUndef() ||
    3950       11056 :                                         isNullConstant(V); });
    3951             :     // TODO: Handle signed overflow.
    3952             :   }
    3953             :   // TODO: Handle oversized shifts.
    3954             :   default:
    3955             :     return false;
    3956             :   }
    3957             : }
    3958             : 
    3959     4534200 : SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
    3960             :                                              EVT VT, SDNode *Cst1,
    3961             :                                              SDNode *Cst2) {
    3962             :   // If the opcode is a target-specific ISD node, there's nothing we can
    3963             :   // do here and the operand rules may not line up with the below, so
    3964             :   // bail early.
    3965     4534200 :   if (Opcode >= ISD::BUILTIN_OP_END)
    3966      337588 :     return SDValue();
    3967             : 
    3968    12589836 :   if (isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)}))
    3969          76 :     return getUNDEF(VT);
    3970             : 
    3971             :   // Handle the case of two scalars.
    3972      442566 :   if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
    3973      432902 :     if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
    3974      432902 :       SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2);
    3975             :       assert((!Folded || !VT.isVector()) &&
    3976             :              "Can't fold vectors ops with scalar operands");
    3977      432902 :       return Folded;
    3978             :     }
    3979             :   }
    3980             : 
    3981             :   // fold (add Sym, c) -> Sym+c
    3982      253677 :   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst1))
    3983      253677 :     return FoldSymbolOffset(Opcode, VT, GA, Cst2);
    3984     3509957 :   if (TLI->isCommutativeBinOp(Opcode))
    3985        1240 :     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst2))
    3986        1240 :       return FoldSymbolOffset(Opcode, VT, GA, Cst1);
    3987             : 
    3988             :   // For vectors extract each constant element into Inputs so we can constant
    3989             :   // fold them individually.
    3990     3508717 :   BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
    3991     3508717 :   BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
    3992     3508717 :   if (!BV1 || !BV2)
    3993     3505189 :     return SDValue();
    3994             : 
    3995             :   assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
    3996             : 
    3997        3528 :   EVT SVT = VT.getScalarType();
    3998        3528 :   EVT LegalSVT = SVT;
    3999        3528 :   if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
    4000        2826 :     LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
    4001        1413 :     if (LegalSVT.bitsLT(SVT))
    4002           0 :       return SDValue();
    4003             :   }
    4004        3528 :   SmallVector<SDValue, 4> Outputs;
    4005       17373 :   for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
    4006       23878 :     SDValue V1 = BV1->getOperand(I);
    4007       23878 :     SDValue V2 = BV2->getOperand(I);
    4008             : 
    4009       11939 :     if (SVT.isInteger()) {
    4010       21968 :         if (V1->getValueType(0).bitsGT(SVT))
    4011         321 :           V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
    4012       21968 :         if (V2->getValueType(0).bitsGT(SVT))
    4013         334 :           V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
    4014             :     }
    4015             : 
    4016       35925 :     if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
    4017           0 :       return SDValue();
    4018             : 
    4019             :     // Fold one vector element.
    4020       11939 :     SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
    4021       11993 :     if (LegalSVT != SVT)
    4022         307 :       ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
    4023             : 
    4024             :     // Scalar folding only succeeded if the result is a constant or UNDEF.
    4025       37944 :     if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
    4026        2135 :         ScalarResult.getOpcode() != ISD::ConstantFP)
    4027        1622 :       return SDValue();
    4028       10317 :     Outputs.push_back(ScalarResult);
    4029             :   }
    4030             : 
    4031             :   assert(VT.getVectorNumElements() == Outputs.size() &&
    4032             :          "Vector size mismatch!");
    4033             : 
    4034             :   // We may have a vector type but a scalar result. Create a splat.
    4035        3812 :   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
    4036             : 
    4037             :   // Build a big vector out of the scalar elements we generated.
    4038        5718 :   return getBuildVector(VT, SDLoc(), Outputs);
    4039             : }
    4040             : 
    4041             : // TODO: Merge with FoldConstantArithmetic
    4042      728700 : SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
    4043             :                                                    const SDLoc &DL, EVT VT,
    4044             :                                                    ArrayRef<SDValue> Ops,
    4045             :                                                    const SDNodeFlags Flags) {
    4046             :   // If the opcode is a target-specific ISD node, there's nothing we can
    4047             :   // do here and the operand rules may not line up with the below, so
    4048             :   // bail early.
    4049      728700 :   if (Opcode >= ISD::BUILTIN_OP_END)
    4050           0 :     return SDValue();
    4051             : 
    4052      728700 :   if (isUndef(Opcode, Ops))
    4053           1 :     return getUNDEF(VT);
    4054             : 
    4055             :   // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
    4056      728699 :   if (!VT.isVector())
    4057      157751 :     return SDValue();
    4058             : 
    4059      570948 :   unsigned NumElts = VT.getVectorNumElements();
    4060             : 
    4061        3427 :   auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
    4062       16738 :     return !Op.getValueType().isVector() ||
    4063        9487 :            Op.getValueType().getVectorNumElements() == NumElts;
    4064      574375 :   };
    4065             : 
    4066      575255 :   auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
    4067      575255 :     BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
    4068     1725624 :     return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
    4069        7473 :            (BV && BV->isConstant());
    4070             :   };
    4071             : 
    4072             :   // All operands must be vector types with the same number of elements as
    4073             :   // the result type and must be either UNDEF or a build vector of constant
    4074             :   // or UNDEF scalars.
    4075     1144215 :   if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
    4076        2319 :       !llvm::all_of(Ops, IsScalarOrSameVectorSize))
    4077      568629 :     return SDValue();
    4078             : 
    4079             :   // If we are comparing vectors, then the result needs to be a i1 boolean
    4080             :   // that is then sign-extended back to the legal result type.
    4081        4241 :   EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
    4082             : 
    4083             :   // Find legal integer scalar type for constant promotion and
    4084             :   // ensure that its scalar size is at least as large as source.
    4085        2319 :   EVT LegalSVT = VT.getScalarType();
    4086        2319 :   if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
    4087        1006 :     LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
    4088         503 :     if (LegalSVT.bitsLT(VT.getScalarType()))
    4089           0 :       return SDValue();
    4090             :   }
    4091             : 
    4092             :   // Constant fold each scalar lane separately.
    4093        2319 :   SmallVector<SDValue, 4> ScalarResults;
    4094       20959 :   for (unsigned i = 0; i != NumElts; i++) {
    4095       37395 :     SmallVector<SDValue, 4> ScalarOps;
    4096       43328 :     for (SDValue Op : Ops) {
    4097       49146 :       EVT InSVT = Op.getValueType().getScalarType();
    4098       22944 :       BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
    4099        1629 :       if (!InBV) {
    4100             :         // We've checked that this is UNDEF or a constant of some kind.
    4101        3258 :         if (Op.isUndef())
    4102         120 :           ScalarOps.push_back(getUNDEF(InSVT));
    4103             :         else
    4104        1509 :           ScalarOps.push_back(Op);
    4105        1629 :         continue;
    4106             :       }
    4107             : 
    4108       45888 :       SDValue ScalarOp = InBV->getOperand(i);
    4109       45888 :       EVT ScalarVT = ScalarOp.getValueType();
    4110             : 
    4111             :       // Build vector (integer) scalar operands may need implicit
    4112             :       // truncation - do this before constant folding.
    4113       22944 :       if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
    4114          96 :         ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
    4115             : 
    4116       22944 :       ScalarOps.push_back(ScalarOp);
    4117             :     }
    4118             : 
    4119             :     // Constant fold the scalar operands.
    4120       18755 :     SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
    4121             : 
    4122             :     // Legalize the (integer) scalar constant if necessary.
    4123       18755 :     if (LegalSVT != SVT)
    4124        5909 :       ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
    4125             : 
    4126             :     // Scalar folding only succeeded if the result is a constant or UNDEF.
    4127       56648 :     if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
    4128         399 :         ScalarResult.getOpcode() != ISD::ConstantFP)
    4129         115 :       return SDValue();
    4130       18640 :     ScalarResults.push_back(ScalarResult);
    4131             :   }
    4132             : 
    4133        2204 :   SDValue V = getBuildVector(VT, DL, ScalarResults);
    4134        2204 :   NewSDValueDbgMsg(V, "New node fold constant vector: ");
    4135        2204 :   return V;
    4136             : }
    4137             : 
    4138     5272890 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4139             :                               SDValue N1, SDValue N2, const SDNodeFlags Flags) {
    4140     5272890 :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    4141     5272890 :   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
    4142     5272890 :   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
    4143     5272890 :   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
    4144             : 
    4145             :   // Canonicalize constant to RHS if commutative.
    4146     5272890 :   if (TLI->isCommutativeBinOp(Opcode)) {
    4147     2321573 :     if (N1C && !N2C) {
    4148             :       std::swap(N1C, N2C);
    4149             :       std::swap(N1, N2);
    4150     2308655 :     } else if (N1CFP && !N2CFP) {
    4151         708 :       std::swap(N1CFP, N2CFP);
    4152             :       std::swap(N1, N2);
    4153             :     }
    4154             :   }
    4155             : 
    4156     5272890 :   switch (Opcode) {
    4157             :   default: break;
    4158     1263402 :   case ISD::TokenFactor:
    4159             :     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
    4160             :            N2.getValueType() == MVT::Other && "Invalid token factor!");
    4161             :     // Fold trivial token factors.
    4162     2526804 :     if (N1.getOpcode() == ISD::EntryToken) return N2;
    4163     2516246 :     if (N2.getOpcode() == ISD::EntryToken) return N1;
    4164         653 :     if (N1 == N2) return N1;
    4165             :     break;
    4166       32247 :   case ISD::CONCAT_VECTORS: {
    4167             :     // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
    4168       32247 :     SDValue Ops[] = {N1, N2};
    4169       32247 :     if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
    4170         480 :       return V;
    4171       31767 :     break;
    4172             :   }
    4173      172893 :   case ISD::AND:
    4174             :     assert(VT.isInteger() && "This operator does not apply to FP types!");
    4175             :     assert(N1.getValueType() == N2.getValueType() &&
    4176             :            N1.getValueType() == VT && "Binary operator types must match!");
    4177             :     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
    4178             :     // worth handling here.
    4179      320000 :     if (N2C && N2C->isNullValue())
    4180        2751 :       return N2;
    4181      458854 :     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
    4182        1840 :       return N1;
    4183             :     break;
    4184     2174558 :   case ISD::OR:
    4185             :   case ISD::XOR:
    4186             :   case ISD::ADD:
    4187             :   case ISD::SUB:
    4188             :     assert(VT.isInteger() && "This operator does not apply to FP types!");
    4189             :     assert(N1.getValueType() == N2.getValueType() &&
    4190             :            N1.getValueType() == VT && "Binary operator types must match!");
    4191             :     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
    4192             :     // it's worth handling here.
    4193     3866160 :     if (N2C && N2C->isNullValue())
    4194      686786 :       return N1;
    4195             :     break;
    4196             :   case ISD::UDIV:
    4197             :   case ISD::UREM:
    4198             :   case ISD::MULHU:
    4199             :   case ISD::MULHS:
    4200             :   case ISD::MUL:
    4201             :   case ISD::SDIV:
    4202             :   case ISD::SREM:
    4203             :   case ISD::SMIN:
    4204             :   case ISD::SMAX:
    4205             :   case ISD::UMIN:
    4206             :   case ISD::UMAX:
    4207             :     assert(VT.isInteger() && "This operator does not apply to FP types!");
    4208             :     assert(N1.getValueType() == N2.getValueType() &&
    4209             :            N1.getValueType() == VT && "Binary operator types must match!");
    4210             :     break;
    4211       33911 :   case ISD::FADD:
    4212             :   case ISD::FSUB:
    4213             :   case ISD::FMUL:
    4214             :   case ISD::FDIV:
    4215             :   case ISD::FREM:
    4216       33911 :     if (getTarget().Options.UnsafeFPMath) {
    4217        5508 :       if (Opcode == ISD::FADD) {
    4218             :         // x+0 --> x
    4219        2099 :         if (N2CFP && N2CFP->getValueAPF().isZero())
    4220          16 :           return N1;
    4221        3935 :       } else if (Opcode == ISD::FSUB) {
    4222             :         // x-0 --> x
    4223         692 :         if (N2CFP && N2CFP->getValueAPF().isZero())
    4224           0 :           return N1;
    4225        3263 :       } else if (Opcode == ISD::FMUL) {
    4226             :         // x*0 --> 0
    4227        3271 :         if (N2CFP && N2CFP->isZero())
    4228           6 :           return N2;
    4229             :         // x*1 --> x
    4230        3259 :         if (N2CFP && N2CFP->isExactlyValue(1.0))
    4231          61 :           return N1;
    4232             :       }
    4233             :     }
    4234             :     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
    4235             :     assert(N1.getValueType() == N2.getValueType() &&
    4236             :            N1.getValueType() == VT && "Binary operator types must match!");
    4237             :     break;
    4238             :   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
    4239             :     assert(N1.getValueType() == VT &&
    4240             :            N1.getValueType().isFloatingPoint() &&
    4241             :            N2.getValueType().isFloatingPoint() &&
    4242             :            "Invalid FCOPYSIGN!");
    4243             :     break;
    4244      365527 :   case ISD::SHL:
    4245             :   case ISD::SRA:
    4246             :   case ISD::SRL:
    4247             :   case ISD::ROTL:
    4248             :   case ISD::ROTR:
    4249             :     assert(VT == N1.getValueType() &&
    4250             :            "Shift operators return type must be the same as their first arg");
    4251             :     assert(VT.isInteger() && N2.getValueType().isInteger() &&
    4252             :            "Shifts only work on integers");
    4253             :     assert((!VT.isVector() || VT == N2.getValueType()) &&
    4254             :            "Vector shift amounts must be in the same as their first arg");
    4255             :     // Verify that the shift amount VT is bit enough to hold valid shift
    4256             :     // amounts.  This catches things like trying to shift an i1024 value by an
    4257             :     // i8, which is easy to fall into in generic code that uses
    4258             :     // TLI.getShiftAmount().
    4259             :     assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) &&
    4260             :            "Invalid use of small shift amount with oversized value!");
    4261             : 
    4262             :     // Always fold shifts of i1 values so the code generator doesn't need to
    4263             :     // handle them.  Since we know the size of the shift has to be less than the
    4264             :     // size of the value, the shift/rotate count is guaranteed to be zero.
    4265      731054 :     if (VT == MVT::i1)
    4266          45 :       return N1;
    4267     1040446 :     if (N2C && N2C->isNullValue())
    4268        8736 :       return N1;
    4269             :     break;
    4270           8 :   case ISD::FP_ROUND_INREG: {
    4271           8 :     EVT EVT = cast<VTSDNode>(N2)->getVT();
    4272             :     assert(VT == N1.getValueType() && "Not an inreg round!");
    4273             :     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
    4274             :            "Cannot FP_ROUND_INREG integer types");
    4275             :     assert(EVT.isVector() == VT.isVector() &&
    4276             :            "FP_ROUND_INREG type should be vector iff the operand "
    4277             :            "type is vector!");
    4278             :     assert((!EVT.isVector() ||
    4279             :             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
    4280             :            "Vector element counts must match in FP_ROUND_INREG");
    4281             :     assert(EVT.bitsLE(VT) && "Not rounding down!");
    4282             :     (void)EVT;
    4283           8 :     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
    4284             :     break;
    4285             :   }
    4286        4836 :   case ISD::FP_ROUND:
    4287             :     assert(VT.isFloatingPoint() &&
    4288             :            N1.getValueType().isFloatingPoint() &&
    4289             :            VT.bitsLE(N1.getValueType()) &&
    4290             :            N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
    4291             :            "Invalid FP_ROUND!");
    4292        9676 :     if (N1.getValueType() == VT) return N1;  // noop conversion.
    4293             :     break;
    4294       56914 :   case ISD::AssertSext:
    4295             :   case ISD::AssertZext: {
    4296       56914 :     EVT EVT = cast<VTSDNode>(N2)->getVT();
    4297             :     assert(VT == N1.getValueType() && "Not an inreg extend!");
    4298             :     assert(VT.isInteger() && EVT.isInteger() &&
    4299             :            "Cannot *_EXTEND_INREG FP types");
    4300             :     assert(!EVT.isVector() &&
    4301             :            "AssertSExt/AssertZExt type should be the vector element type "
    4302             :            "rather than the vector type!");
    4303             :     assert(EVT.bitsLE(VT) && "Not extending!");
    4304       57292 :     if (VT == EVT) return N1; // noop assertion.
    4305             :     break;
    4306             :   }
    4307       18102 :   case ISD::SIGN_EXTEND_INREG: {
    4308       18102 :     EVT EVT = cast<VTSDNode>(N2)->getVT();
    4309             :     assert(VT == N1.getValueType() && "Not an inreg extend!");
    4310             :     assert(VT.isInteger() && EVT.isInteger() &&
    4311             :            "Cannot *_EXTEND_INREG FP types");
    4312             :     assert(EVT.isVector() == VT.isVector() &&
    4313             :            "SIGN_EXTEND_INREG type should be vector iff the operand "
    4314             :            "type is vector!");
    4315             :     assert((!EVT.isVector() ||
    4316             :             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
    4317             :            "Vector element counts must match in SIGN_EXTEND_INREG");
    4318             :     assert(EVT.bitsLE(VT) && "Not extending!");
    4319       18729 :     if (EVT == VT) return N1;  // Not actually extending
    4320             : 
    4321        1639 :     auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
    4322        3278 :       unsigned FromBits = EVT.getScalarSizeInBits();
    4323        1639 :       Val <<= Val.getBitWidth() - FromBits;
    4324        1639 :       Val.ashrInPlace(Val.getBitWidth() - FromBits);
    4325        1639 :       return getConstant(Val, DL, ConstantVT);
    4326       18012 :     };
    4327             : 
    4328       18012 :     if (N1C) {
    4329         262 :       const APInt &Val = N1C->getAPIntValue();
    4330         262 :       return SignExtendInReg(Val, VT);
    4331             :     }
    4332       17881 :     if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
    4333         642 :       SmallVector<SDValue, 8> Ops;
    4334         963 :       llvm::EVT OpVT = N1.getOperand(0).getValueType();
    4335        1839 :       for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
    4336        3036 :         SDValue Op = N1.getOperand(i);
    4337        3046 :         if (Op.isUndef()) {
    4338          10 :           Ops.push_back(getUNDEF(OpVT));
    4339          10 :           continue;
    4340             :         }
    4341        1508 :         ConstantSDNode *C = cast<ConstantSDNode>(Op);
    4342        4524 :         APInt Val = C->getAPIntValue();
    4343        3016 :         Ops.push_back(SignExtendInReg(Val, OpVT));
    4344             :       }
    4345         321 :       return getBuildVector(VT, DL, Ops);
    4346             :     }
    4347       17560 :     break;
    4348             :   }
    4349      319478 :   case ISD::EXTRACT_VECTOR_ELT:
    4350             :     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
    4351      638956 :     if (N1.isUndef())
    4352       12445 :       return getUNDEF(VT);
    4353             : 
    4354             :     // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
    4355      918791 :     if (N2C && N2C->getZExtValue() >= N1.getValueType().getVectorNumElements())
    4356          84 :       return getUNDEF(VT);
    4357             : 
    4358             :     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
    4359             :     // expanding copies of large vectors from registers.
    4360      305753 :     if (N2C &&
    4361      616317 :         N1.getOpcode() == ISD::CONCAT_VECTORS &&
    4362        3615 :         N1.getNumOperands() > 0) {
    4363             :       unsigned Factor =
    4364       10845 :         N1.getOperand(0).getValueType().getVectorNumElements();
    4365             :       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
    4366       10845 :                      N1.getOperand(N2C->getZExtValue() / Factor),
    4367        3615 :                      getConstant(N2C->getZExtValue() % Factor, DL,
    4368       10845 :                                  N2.getValueType()));
    4369             :     }
    4370             : 
    4371             :     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
    4372             :     // expanding large vector constants.
    4373      907610 :     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
    4374      310554 :       SDValue Elt = N1.getOperand(N2C->getZExtValue());
    4375             : 
    4376      207072 :       if (VT != Elt.getValueType())
    4377             :         // If the vector element type is not legal, the BUILD_VECTOR operands
    4378             :         // are promoted and implicitly truncated, and the result implicitly
    4379             :         // extended. Make that explicit here.
    4380        6167 :         Elt = getAnyExtOrTrunc(Elt, DL, VT);
    4381             : 
    4382      103518 :       return Elt;
    4383             :     }
    4384             : 
    4385             :     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
    4386             :     // operations are lowered to scalars.
    4387      399632 :     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
    4388             :       // If the indices are the same, return the inserted element else
    4389             :       // if the indices are known different, extract the element from
    4390             :       // the original vector.
    4391        8078 :       SDValue N1Op2 = N1.getOperand(2);
    4392        2928 :       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
    4393             : 
    4394        2928 :       if (N1Op2C && N2C) {
    4395        5844 :         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
    4396        1341 :           if (VT == N1.getOperand(1).getValueType())
    4397        3782 :             return N1.getOperand(1);
    4398             :           else
    4399          34 :             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
    4400             :         }
    4401             : 
    4402        4950 :         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
    4403             :       }
    4404             :     }
    4405             : 
    4406             :     // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
    4407             :     // when vector types are scalarized and v1iX is legal.
    4408             :     // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx)
    4409      407353 :     if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
    4410      223998 :         N1.getValueType().getVectorNumElements() == 1) {
    4411          52 :       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
    4412          78 :                      N1.getOperand(1));
    4413             :     }
    4414             :     break;
    4415       33438 :   case ISD::EXTRACT_ELEMENT:
    4416             :     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
    4417             :     assert(!N1.getValueType().isVector() && !VT.isVector() &&
    4418             :            (N1.getValueType().isInteger() == VT.isInteger()) &&
    4419             :            N1.getValueType() != VT &&
    4420             :            "Wrong types for EXTRACT_ELEMENT!");
    4421             : 
    4422             :     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
    4423             :     // 64-bit integers into 32-bit parts.  Instead of building the extract of
    4424             :     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
    4425       66876 :     if (N1.getOpcode() == ISD::BUILD_PAIR)
    4426       17202 :       return N1.getOperand(N2C->getZExtValue());
    4427             : 
    4428             :     // EXTRACT_ELEMENT of a constant int is also very common.
    4429       27704 :     if (N1C) {
    4430        5618 :       unsigned ElementSize = VT.getSizeInBits();
    4431       11236 :       unsigned Shift = ElementSize * N2C->getZExtValue();
    4432       16854 :       APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
    4433       11236 :       return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
    4434       22086 :     }
    4435             :     break;
    4436       98762 :   case ISD::EXTRACT_SUBVECTOR:
    4437      196795 :     if (VT.isSimple() && N1.getValueType().isSimple()) {
    4438             :       assert(VT.isVector() && N1.getValueType().isVector() &&
    4439             :              "Extract subvector VTs must be a vectors!");
    4440             :       assert(VT.getVectorElementType() ==
    4441             :              N1.getValueType().getVectorElementType() &&
    4442             :              "Extract subvector VTs must have the same element type!");
    4443             :       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
    4444             :              "Extract subvector must be from larger vector to smaller vector!");
    4445             : 
    4446       93601 :       if (N2C) {
    4447             :         assert((VT.getVectorNumElements() + N2C->getZExtValue()
    4448             :                 <= N1.getValueType().getVectorNumElements())
    4449             :                && "Extract subvector overflow!");
    4450             :       }
    4451             : 
    4452             :       // Trivial extraction.
    4453       93601 :       if (VT.getSimpleVT() == N1.getSimpleValueType())
    4454       14600 :         return N1;
    4455             : 
    4456             :       // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
    4457      158002 :       if (N1.isUndef())
    4458         639 :         return getUNDEF(VT);
    4459             : 
    4460             :       // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
    4461             :       // the concat have the same type as the extract.
    4462      165448 :       if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
    4463       87086 :           N1.getNumOperands() > 0 &&
    4464       33133 :           VT == N1.getOperand(0).getValueType()) {
    4465        6961 :         unsigned Factor = VT.getVectorNumElements();
    4466       20883 :         return N1.getOperand(N2C->getZExtValue() / Factor);
    4467             :       }
    4468             : 
    4469             :       // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
    4470             :       // during shuffle legalization.
    4471      143472 :       if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
    4472         736 :           VT == N1.getOperand(1).getValueType())
    4473         368 :         return N1.getOperand(1);
    4474             :     }
    4475             :     break;
    4476             :   }
    4477             : 
    4478             :   // Perform trivial constant folding.
    4479     4381675 :   if (SDValue SV =
    4480     4381675 :           FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
    4481      183520 :     return SV;
    4482             : 
    4483             :   // Constant fold FP operations.
    4484     4198155 :   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
    4485     4198155 :   if (N1CFP) {
    4486        3284 :     if (N2CFP) {
    4487       12996 :       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
    4488             :       APFloat::opStatus s;
    4489        2036 :       switch (Opcode) {
    4490         305 :       case ISD::FADD:
    4491         305 :         s = V1.add(V2, APFloat::rmNearestTiesToEven);
    4492         305 :         if (!HasFPExceptions || s != APFloat::opInvalidOp)
    4493         933 :           return getConstantFP(V1, DL, VT);
    4494             :         break;
    4495          27 :       case ISD::FSUB:
    4496          27 :         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
    4497          27 :         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
    4498          27 :           return getConstantFP(V1, DL, VT);
    4499             :         break;
    4500         263 :       case ISD::FMUL:
    4501         263 :         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
    4502         263 :         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
    4503         263 :           return getConstantFP(V1, DL, VT);
    4504             :         break;
    4505          39 :       case ISD::FDIV:
    4506          39 :         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
    4507          39 :         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
    4508             :                                  s!=APFloat::opDivByZero)) {
    4509          18 :           return getConstantFP(V1, DL, VT);
    4510             :         }
    4511             :         break;
    4512          25 :       case ISD::FREM :
    4513          25 :         s = V1.mod(V2);
    4514          25 :         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
    4515             :                                  s!=APFloat::opDivByZero)) {
    4516          11 :           return getConstantFP(V1, DL, VT);
    4517             :         }
    4518             :         break;
    4519           4 :       case ISD::FCOPYSIGN:
    4520           4 :         V1.copySign(V2);
    4521           4 :         return getConstantFP(V1, DL, VT);
    4522             :       default: break;
    4523             :       }
    4524             :     }
    4525             : 
    4526        2656 :     if (Opcode == ISD::FP_ROUND) {
    4527          16 :       APFloat V = N1CFP->getValueAPF();    // make copy
    4528             :       bool ignored;
    4529             :       // This can return overflow, underflow, or inexact; we don't care.
    4530             :       // FIXME need to be more flexible about rounding mode.
    4531           4 :       (void)V.convert(EVTToAPFloatSemantics(VT),
    4532             :                       APFloat::rmNearestTiesToEven, &ignored);
    4533           4 :       return getConstantFP(V, DL, VT);
    4534             :     }
    4535             :   }
    4536             : 
    4537             :   // Canonicalize an UNDEF to the RHS, even over a constant.
    4538     8395046 :   if (N1.isUndef()) {
    4539        4129 :     if (TLI->isCommutativeBinOp(Opcode)) {
    4540             :       std::swap(N1, N2);
    4541             :     } else {
    4542        1473 :       switch (Opcode) {
    4543          47 :       case ISD::FP_ROUND_INREG:
    4544             :       case ISD::SIGN_EXTEND_INREG:
    4545             :       case ISD::SUB:
    4546             :       case ISD::FSUB:
    4547             :       case ISD::FDIV:
    4548             :       case ISD::FREM:
    4549             :       case ISD::SRA:
    4550          47 :         return N1;     // fold op(undef, arg2) -> undef
    4551         218 :       case ISD::UDIV:
    4552             :       case ISD::SDIV:
    4553             :       case ISD::UREM:
    4554             :       case ISD::SREM:
    4555             :       case ISD::SRL:
    4556             :       case ISD::SHL:
    4557         218 :         if (!VT.isVector())
    4558         203 :           return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
    4559             :         // For vectors, we can't easily build an all zero vector, just return
    4560             :         // the LHS.
    4561          15 :         return N2;
    4562             :       }
    4563             :     }
    4564             :   }
    4565             : 
    4566             :   // Fold a bunch of operators when the RHS is undef.
    4567     8394516 :   if (N2.isUndef()) {
    4568       11901 :     switch (Opcode) {
    4569         477 :     case ISD::XOR:
    4570         954 :       if (N1.isUndef())
    4571             :         // Handle undef ^ undef -> 0 special case. This is a common
    4572             :         // idiom (misuse).
    4573           6 :         return getConstant(0, DL, VT);
    4574             :       LLVM_FALLTHROUGH;
    4575             :     case ISD::ADD:
    4576             :     case ISD::ADDC:
    4577             :     case ISD::ADDE:
    4578             :     case ISD::SUB:
    4579             :     case ISD::UDIV:
    4580             :     case ISD::SDIV:
    4581             :     case ISD::UREM:
    4582             :     case ISD::SREM:
    4583        1893 :       return N2;       // fold op(arg1, undef) -> undef
    4584        1285 :     case ISD::FADD:
    4585             :     case ISD::FSUB:
    4586             :     case ISD::FMUL:
    4587             :     case ISD::FDIV:
    4588             :     case ISD::FREM:
    4589        1285 :       if (getTarget().Options.UnsafeFPMath)
    4590           5 :         return N2;
    4591             :       break;
    4592         166 :     case ISD::MUL:
    4593             :     case ISD::AND:
    4594             :     case ISD::SRL:
    4595             :     case ISD::SHL:
    4596         166 :       if (!VT.isVector())
    4597          97 :         return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
    4598             :       // For vectors, we can't easily build an all zero vector, just return
    4599             :       // the LHS.
    4600          69 :       return N1;
    4601          81 :     case ISD::OR:
    4602          81 :       if (!VT.isVector())
    4603          82 :         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT);
    4604             :       // For vectors, we can't easily build an all one vector, just return
    4605             :       // the LHS.
    4606          40 :       return N1;
    4607           6 :     case ISD::SRA:
    4608           6 :       return N1;
    4609             :     }
    4610             :   }
    4611             : 
    4612             :   // Memoize this node if possible.
    4613             :   SDNode *N;
    4614     4195101 :   SDVTList VTs = getVTList(VT);
    4615     4195101 :   SDValue Ops[] = {N1, N2};
    4616     4195101 :   if (VT != MVT::Glue) {
    4617     7855317 :     FoldingSetNodeID ID;
    4618     4191948 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    4619     4191948 :     void *IP = nullptr;
    4620     4191948 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
    4621      528579 :       E->intersectFlagsWith(Flags);
    4622      528579 :       return SDValue(E, 0);
    4623             :     }
    4624             : 
    4625     7326738 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4626     7326738 :     N->setFlags(Flags);
    4627     3663369 :     createOperands(N, Ops);
    4628     7326738 :     CSEMap.InsertNode(N, IP);
    4629             :   } else {
    4630        6306 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4631        3153 :     createOperands(N, Ops);
    4632             :   }
    4633             : 
    4634     3666522 :   InsertNode(N);
    4635     3666522 :   SDValue V = SDValue(N, 0);
    4636     3666522 :   NewSDValueDbgMsg(V, "Creating new node: ");
    4637     3666522 :   return V;
    4638             : }
    4639             : 
    4640      772646 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4641             :                               SDValue N1, SDValue N2, SDValue N3) {
    4642             :   // Perform various simplifications.
    4643      772646 :   switch (Opcode) {
    4644        3537 :   case ISD::FMA: {
    4645        3537 :     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
    4646        3537 :     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
    4647        3537 :     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
    4648        3537 :     if (N1CFP && N2CFP && N3CFP) {
    4649          16 :       APFloat  V1 = N1CFP->getValueAPF();
    4650           8 :       const APFloat &V2 = N2CFP->getValueAPF();
    4651           8 :       const APFloat &V3 = N3CFP->getValueAPF();
    4652             :       APFloat::opStatus s =
    4653           8 :         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
    4654           8 :       if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
    4655          16 :         return getConstantFP(V1, DL, VT);
    4656             :     }
    4657             :     break;
    4658             :   }
    4659          22 :   case ISD::CONCAT_VECTORS: {
    4660             :     // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
    4661          22 :     SDValue Ops[] = {N1, N2, N3};
    4662          22 :     if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
    4663           2 :       return V;
    4664          20 :     break;
    4665             :   }
    4666      186805 :   case ISD::SETCC: {
    4667             :     // Use FoldSetCC to simplify SETCC's.
    4668      186805 :     if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
    4669        2162 :       return V;
    4670             :     // Vector constant folding.
    4671      184643 :     SDValue Ops[] = {N1, N2, N3};
    4672      369286 :     if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
    4673         340 :       NewSDValueDbgMsg(V, "New node vector constant folding: ");
    4674         340 :       return V;
    4675             :     }
    4676      184303 :     break;
    4677             :   }
    4678       69218 :   case ISD::SELECT:
    4679         399 :     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
    4680         399 :      if (N1C->getZExtValue())
    4681         137 :        return N2;             // select true, X, Y -> X
    4682         262 :      return N3;             // select false, X, Y -> Y
    4683             :     }
    4684             : 
    4685          84 :     if (N2 == N3) return N2;   // select C, X, X -> X
    4686             :     break;
    4687           0 :   case ISD::VECTOR_SHUFFLE:
    4688           0 :     llvm_unreachable("should use getVectorShuffle constructor!");
    4689       34108 :   case ISD::INSERT_VECTOR_ELT: {
    4690       33858 :     ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
    4691             :     // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF
    4692       67716 :     if (N3C && N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
    4693          12 :       return getUNDEF(VT);
    4694             :     break;
    4695             :   }
    4696       14966 :   case ISD::INSERT_SUBVECTOR: {
    4697       14966 :     SDValue Index = N3;
    4698       44898 :     if (VT.isSimple() && N1.getValueType().isSimple()
    4699       44898 :         && N2.getValueType().isSimple()) {
    4700             :       assert(VT.isVector() && N1.getValueType().isVector() &&
    4701             :              N2.getValueType().isVector() &&
    4702             :              "Insert subvector VTs must be a vectors");
    4703             :       assert(VT == N1.getValueType() &&
    4704             :              "Dest and insert subvector source types must match!");
    4705             :       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
    4706             :              "Insert subvector must be from smaller vector to larger vector!");
    4707       14966 :       if (isa<ConstantSDNode>(Index)) {
    4708             :         assert((N2.getValueType().getVectorNumElements() +
    4709             :                 cast<ConstantSDNode>(Index)->getZExtValue()
    4710             :                 <= VT.getVectorNumElements())
    4711             :                && "Insert subvector overflow!");
    4712             :       }
    4713             : 
    4714             :       // Trivial insertion.
    4715       14966 :       if (VT.getSimpleVT() == N2.getSimpleValueType())
    4716          98 :         return N2;
    4717             :     }
    4718       14868 :     break;
    4719             :   }
    4720           0 :   case ISD::BITCAST:
    4721             :     // Fold bit_convert nodes from a type to themselves.
    4722           0 :     if (N1.getValueType() == VT)
    4723           0 :       return N1;
    4724             :     break;
    4725             :   }
    4726             : 
    4727             :   // Memoize node if it doesn't produce a flag.
    4728             :   SDNode *N;
    4729      769541 :   SDVTList VTs = getVTList(VT);
    4730      769541 :   SDValue Ops[] = {N1, N2, N3};
    4731      769541 :   if (VT != MVT::Glue) {
    4732     1442903 :     FoldingSetNodeID ID;
    4733      767607 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    4734      767607 :     void *IP = nullptr;
    4735      767607 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    4736       92311 :       return SDValue(E, 0);
    4737             : 
    4738     1350592 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4739      675296 :     createOperands(N, Ops);
    4740     1350592 :     CSEMap.InsertNode(N, IP);
    4741             :   } else {
    4742        3868 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    4743        1934 :     createOperands(N, Ops);
    4744             :   }
    4745             : 
    4746      677230 :   InsertNode(N);
    4747      677230 :   SDValue V = SDValue(N, 0);
    4748      677230 :   NewSDValueDbgMsg(V, "Creating new node: ");
    4749      677230 :   return V;
    4750             : }
    4751             : 
    4752       36207 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4753             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
    4754       36207 :   SDValue Ops[] = { N1, N2, N3, N4 };
    4755       72414 :   return getNode(Opcode, DL, VT, Ops);
    4756             : }
    4757             : 
    4758       39171 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    4759             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
    4760             :                               SDValue N5) {
    4761       39171 :   SDValue Ops[] = { N1, N2, N3, N4, N5 };
    4762       78342 :   return getNode(Opcode, DL, VT, Ops);
    4763             : }
    4764             : 
    4765             : /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
    4766             : /// the incoming stack arguments to be loaded from the stack.
    4767         154 : SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
    4768         308 :   SmallVector<SDValue, 8> ArgChains;
    4769             : 
    4770             :   // Include the original chain at the beginning of the list. When this is
    4771             :   // used by target LowerCall hooks, this helps legalize find the
    4772             :   // CALLSEQ_BEGIN node.
    4773         154 :   ArgChains.push_back(Chain);
    4774             : 
    4775             :   // Add a chain value for each stack argument.
    4776         308 :   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
    4777        1200 :        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
    4778         737 :     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
    4779         255 :       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
    4780         102 :         if (FI->getIndex() < 0)
    4781         102 :           ArgChains.push_back(SDValue(L, 1));
    4782             : 
    4783             :   // Build a tokenfactor for all the chains.
    4784         924 :   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
    4785             : }
    4786             : 
    4787             : /// getMemsetValue - Vectorized representation of the memset value
    4788             : /// operand.
    4789       22023 : static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
    4790             :                               const SDLoc &dl) {
    4791             :   assert(!Value.isUndef());
    4792             : 
    4793       22023 :   unsigned NumBits = VT.getScalarSizeInBits();
    4794       21989 :   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
    4795             :     assert(C->getAPIntValue().getBitWidth() == 8);
    4796       43978 :     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
    4797       21989 :     if (VT.isInteger())
    4798       20313 :       return DAG.getConstant(Val, dl, VT);
    4799        5028 :     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
    4800        1676 :                              VT);
    4801             :   }
    4802             : 
    4803             :   assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
    4804          34 :   EVT IntVT = VT.getScalarType();
    4805          34 :   if (!IntVT.isInteger())
    4806           1 :     IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
    4807             : 
    4808          34 :   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
    4809          34 :   if (NumBits > 8) {
    4810             :     // Use a multiplication with 0x010101... to extend the input to the
    4811             :     // required length.
    4812          54 :     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
    4813          18 :     Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
    4814          36 :                         DAG.getConstant(Magic, dl, IntVT));
    4815             :   }
    4816             : 
    4817          68 :   if (VT != Value.getValueType() && !VT.isInteger())
    4818           1 :     Value = DAG.getBitcast(VT.getScalarType(), Value);
    4819          68 :   if (VT != Value.getValueType())
    4820          18 :     Value = DAG.getSplatBuildVector(VT, dl, Value);
    4821             : 
    4822          34 :   return Value;
    4823             : }
    4824             : 
    4825             : /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
    4826             : /// used when a memcpy is turned into a memset when the source is a constant
    4827             : /// string ptr.
    4828         311 : static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
    4829             :                                   const TargetLowering &TLI,
    4830             :                                   const ConstantDataArraySlice &Slice) {
    4831             :   // Handle vector with all elements zero.
    4832         311 :   if (Slice.Array == nullptr) {
    4833          45 :     if (VT.isInteger())
    4834          42 :       return DAG.getConstant(0, dl, VT);
    4835           9 :     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
    4836           3 :       return DAG.getConstantFP(0.0, dl, VT);
    4837           0 :     else if (VT.isVector()) {
    4838           0 :       unsigned NumElts = VT.getVectorNumElements();
    4839           0 :       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
    4840             :       return DAG.getNode(ISD::BITCAST, dl, VT,
    4841             :                          DAG.getConstant(0, dl,
    4842           0 :                                          EVT::getVectorVT(*DAG.getContext(),
    4843           0 :                                                           EltVT, NumElts)));
    4844             :     } else
    4845           0 :       llvm_unreachable("Expected type!");
    4846             :   }
    4847             : 
    4848             :   assert(!VT.isVector() && "Can't handle vector type here!");
    4849         266 :   unsigned NumVTBits = VT.getSizeInBits();
    4850         266 :   unsigned NumVTBytes = NumVTBits / 8;
    4851         532 :   unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
    4852             : 
    4853         266 :   APInt Val(NumVTBits, 0);
    4854         532 :   if (DAG.getDataLayout().isLittleEndian()) {
    4855        2144 :     for (unsigned i = 0; i != NumBytes; ++i)
    4856        2817 :       Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
    4857             :   } else {
    4858           0 :     for (unsigned i = 0; i != NumBytes; ++i)
    4859           0 :       Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
    4860             :   }
    4861             : 
    4862             :   // If the "cost" of materializing the integer immediate is less than the cost
    4863             :   // of a load, then it is cost effective to turn the load into the immediate.
    4864         266 :   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
    4865         266 :   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
    4866         251 :     return DAG.getConstant(Val, dl, VT);
    4867          15 :   return SDValue(nullptr, 0);
    4868             : }
    4869             : 
    4870       46979 : SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset,
    4871             :                                            const SDLoc &DL) {
    4872       93958 :   EVT VT = Base.getValueType();
    4873       46979 :   return getNode(ISD::ADD, DL, VT, Base, getConstant(Offset, DL, VT));
    4874             : }
    4875             : 
    4876             : /// Returns true if memcpy source is constant data.
    4877        1423 : static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
    4878        1423 :   uint64_t SrcDelta = 0;
    4879        1423 :   GlobalAddressSDNode *G = nullptr;
    4880        2846 :   if (Src.getOpcode() == ISD::GlobalAddress)
    4881             :     G = cast<GlobalAddressSDNode>(Src);
    4882        1221 :   else if (Src.getOpcode() == ISD::ADD &&
    4883        1231 :            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
    4884          20 :            Src.getOperand(1).getOpcode() == ISD::Constant) {
    4885          27 :     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
    4886          27 :     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
    4887             :   }
    4888         298 :   if (!G)
    4889             :     return false;
    4890             : 
    4891         298 :   return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
    4892         596 :                                   SrcDelta + G->getOffset());
    4893             : }
    4894             : 
    4895             : /// Determines the optimal series of memory ops to replace the memset / memcpy.
    4896             : /// Return true if the number of memory ops is below the threshold (Limit).
    4897             : /// It returns the types of the sequence of memory ops to perform
    4898             : /// memset / memcpy by reference.
    4899       22952 : static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
    4900             :                                      unsigned Limit, uint64_t Size,
    4901             :                                      unsigned DstAlign, unsigned SrcAlign,
    4902             :                                      bool IsMemset,
    4903             :                                      bool ZeroMemset,
    4904             :                                      bool MemcpyStrSrc,
    4905             :                                      bool AllowOverlap,
    4906             :                                      unsigned DstAS, unsigned SrcAS,
    4907             :                                      SelectionDAG &DAG,
    4908             :                                      const TargetLowering &TLI) {
    4909             :   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
    4910             :          "Expecting memcpy / memset source to meet alignment requirement!");
    4911             :   // If 'SrcAlign' is zero, that means the memory operation does not need to
    4912             :   // load the value, i.e. memset or memcpy from constant string. Otherwise,
    4913             :   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
    4914             :   // is the specified alignment of the memory operation. If it is zero, that
    4915             :   // means it's possible to change the alignment of the destination.
    4916             :   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
    4917             :   // not need to be loaded.
    4918             :   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
    4919             :                                    IsMemset, ZeroMemset, MemcpyStrSrc,
    4920       22952 :                                    DAG.getMachineFunction());
    4921             : 
    4922       23419 :   if (VT == MVT::Other) {
    4923             :     // Use the largest integer type whose alignment constraints are satisfied.
    4924             :     // We only need to check DstAlign here as SrcAlign is always greater or
    4925             :     // equal to DstAlign (or zero).
    4926         467 :     VT = MVT::i64;
    4927        1691 :     while (DstAlign && DstAlign < VT.getSizeInBits() / 8 &&
    4928         584 :            !TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign))
    4929         640 :       VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
    4930             :     assert(VT.isInteger());
    4931             : 
    4932             :     // Find the largest legal integer type.
    4933             :     MVT LVT = MVT::i64;
    4934        2379 :     while (!TLI.isTypeLegal(LVT))
    4935         326 :       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
    4936             :     assert(LVT.isInteger());
    4937             : 
    4938             :     // If the type we've chosen is larger than the largest legal integer type
    4939             :     // then use that instead.
    4940         467 :     if (VT.bitsGT(LVT))
    4941          63 :       VT = LVT;
    4942             :   }
    4943             : 
    4944             :   unsigned NumMemOps = 0;
    4945      138556 :   while (Size != 0) {
    4946       59073 :     unsigned VTSize = VT.getSizeInBits() / 8;
    4947       63597 :     while (VTSize > Size) {
    4948             :       // For now, only use non-vector load / store's for the left-over pieces.
    4949        2262 :       EVT NewVT = VT;
    4950             :       unsigned NewVTSize;
    4951             : 
    4952        2262 :       bool Found = false;
    4953        2262 :       if (VT.isVector() || VT.isFloatingPoint()) {
    4954        3368 :         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
    4955        2665 :         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
    4956         981 :             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
    4957             :           Found = true;
    4958         703 :         else if (NewVT == MVT::i64 &&
    4959        2051 :                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
    4960        1349 :                  TLI.isSafeMemOpType(MVT::f64)) {
    4961             :           // i64 is usually not legal on 32-bit targets, but f64 may be.
    4962         644 :           NewVT = MVT::f64;
    4963         644 :           Found = true;
    4964             :         }
    4965             :       }
    4966             : 
    4967             :       if (!Found) {
    4968             :         do {
    4969        1274 :           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
    4970        1132 :           if (NewVT == MVT::i8)
    4971             :             break;
    4972         495 :         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
    4973             :       }
    4974        2262 :       NewVTSize = NewVT.getSizeInBits() / 8;
    4975             : 
    4976             :       // If the new VT cannot cover all of the remaining bits, then consider
    4977             :       // issuing a (or a pair of) unaligned and overlapping load / store.
    4978             :       // FIXME: Only does this for 64-bit or more since we don't have proper
    4979             :       // cost model for unaligned load / store.
    4980             :       bool Fast;
    4981        4288 :       if (NumMemOps && AllowOverlap &&
    4982        1869 :           VTSize >= 8 && NewVTSize < Size &&
    4983        2438 :           TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast) && Fast)
    4984          79 :         VTSize = Size;
    4985             :       else {
    4986        2183 :         VT = NewVT;
    4987        2183 :         VTSize = NewVTSize;
    4988             :       }
    4989             :     }
    4990             : 
    4991       59073 :     if (++NumMemOps > Limit)
    4992             :       return false;
    4993             : 
    4994       57802 :     MemOps.push_back(VT);
    4995       57802 :     Size -= VTSize;
    4996             :   }
    4997             : 
    4998             :   return true;
    4999             : }
    5000             : 
    5001       22952 : static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
    5002             :   // On Darwin, -Os means optimize for size without hurting performance, so
    5003             :   // only really optimize for size when -Oz (MinSize) is used.
    5004       45766 :   if (MF.getTarget().getTargetTriple().isOSDarwin())
    5005         276 :     return MF.getFunction()->optForMinSize();
    5006       22814 :   return MF.getFunction()->optForSize();
    5007             : }
    5008             : 
    5009        1426 : static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
    5010             :                                        SDValue Chain, SDValue Dst, SDValue Src,
    5011             :                                        uint64_t Size, unsigned Align,
    5012             :                                        bool isVol, bool AlwaysInline,
    5013             :                                        MachinePointerInfo DstPtrInfo,
    5014             :                                        MachinePointerInfo SrcPtrInfo) {
    5015             :   // Turn a memcpy of undef to nop.
    5016        2852 :   if (Src.isUndef())
    5017           3 :     return Chain;
    5018             : 
    5019             :   // Expand memcpy to a series of load and store ops if the size operand falls
    5020             :   // below a certain threshold.
    5021             :   // TODO: In the AlwaysInline case, if the size is big then generate a loop
    5022             :   // rather than maybe a humongous number of loads and stores.
    5023        1423 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5024        2846 :   const DataLayout &DL = DAG.getDataLayout();
    5025        1423 :   LLVMContext &C = *DAG.getContext();
    5026        1423 :   std::vector<EVT> MemOps;
    5027        1423 :   bool DstAlignCanChange = false;
    5028        1423 :   MachineFunction &MF = DAG.getMachineFunction();
    5029        1423 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    5030        1423 :   bool OptSize = shouldLowerMemFuncForSize(MF);
    5031         254 :   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
    5032         254 :   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
    5033             :     DstAlignCanChange = true;
    5034        1423 :   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
    5035        1423 :   if (Align > SrcAlign)
    5036         532 :     SrcAlign = Align;
    5037             :   ConstantDataArraySlice Slice;
    5038        1423 :   bool CopyFromConstant = isMemSrcFromConstant(Src, Slice);
    5039        1423 :   bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
    5040        2846 :   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
    5041             : 
    5042        1423 :   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
    5043             :                                 (DstAlignCanChange ? 0 : Align),
    5044             :                                 (isZeroConstant ? 0 : SrcAlign),
    5045             :                                 false, false, CopyFromConstant, true,
    5046             :                                 DstPtrInfo.getAddrSpace(),
    5047             :                                 SrcPtrInfo.getAddrSpace(),
    5048             :                                 DAG, TLI))
    5049         273 :     return SDValue();
    5050             : 
    5051        1150 :   if (DstAlignCanChange) {
    5052         233 :     Type *Ty = MemOps[0].getTypeForEVT(C);
    5053         233 :     unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
    5054             : 
    5055             :     // Don't promote to an alignment that would require dynamic stack
    5056             :     // realignment.
    5057         233 :     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    5058         233 :     if (!TRI->needsStackRealignment(MF))
    5059         232 :       while (NewAlign > Align &&
    5060         152 :              DL.exceedsNaturalStackAlignment(NewAlign))
    5061           3 :           NewAlign /= 2;
    5062             : 
    5063         233 :     if (NewAlign > Align) {
    5064             :       // Give the stack frame object a larger alignment if needed.
    5065         302 :       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
    5066          59 :         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
    5067             :       Align = NewAlign;
    5068             :     }
    5069             :   }
    5070             : 
    5071        1150 :   MachineMemOperand::Flags MMOFlags =
    5072        1150 :       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
    5073        1150 :   SmallVector<SDValue, 8> OutChains;
    5074        2300 :   unsigned NumMemOps = MemOps.size();
    5075        1150 :   uint64_t SrcOff = 0, DstOff = 0;
    5076        3476 :   for (unsigned i = 0; i != NumMemOps; ++i) {
    5077        4652 :     EVT VT = MemOps[i];
    5078        2326 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5079        2326 :     SDValue Value, Store;
    5080             : 
    5081        2326 :     if (VTSize > Size) {
    5082             :       // Issuing an unaligned load / store pair  that overlaps with the previous
    5083             :       // pair. Adjust the offset accordingly.
    5084             :       assert(i == NumMemOps-1 && i != 0);
    5085          76 :       SrcOff -= VTSize - Size;
    5086          76 :       DstOff -= VTSize - Size;
    5087             :     }
    5088             : 
    5089        2326 :     if (CopyFromConstant &&
    5090         684 :         (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
    5091             :       // It's unlikely a store of a vector immediate can be done in a single
    5092             :       // instruction. It would require a load from a constantpool first.
    5093             :       // We only handle zero vectors here.
    5094             :       // FIXME: Handle other cases where store of vector immediate is done in
    5095             :       // a single instruction.
    5096             :       ConstantDataArraySlice SubSlice;
    5097         311 :       if (SrcOff < Slice.Length) {
    5098         267 :         SubSlice = Slice;
    5099         267 :         SubSlice.move(SrcOff);
    5100             :       } else {
    5101             :         // This is an out-of-bounds access and hence UB. Pretend we read zero.
    5102          44 :         SubSlice.Array = nullptr;
    5103          44 :         SubSlice.Offset = 0;
    5104          44 :         SubSlice.Length = VTSize;
    5105             :       }
    5106         311 :       Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
    5107         311 :       if (Value.getNode())
    5108         296 :         Store = DAG.getStore(Chain, dl, Value,
    5109             :                              DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5110             :                              DstPtrInfo.getWithOffset(DstOff), Align,
    5111         888 :                              MMOFlags);
    5112             :     }
    5113             : 
    5114        2326 :     if (!Store.getNode()) {
    5115             :       // The type might not be legal for the target.  This should only happen
    5116             :       // if the type is smaller than a legal type, as on PPC, so the right
    5117             :       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
    5118             :       // to Load/Store if NVT==VT.
    5119             :       // FIXME does the case above also need this?
    5120        4060 :       EVT NVT = TLI.getTypeToTransformTo(C, VT);
    5121             :       assert(NVT.bitsGE(VT));
    5122             : 
    5123             :       bool isDereferenceable =
    5124        4060 :         SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
    5125        2030 :       MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
    5126        2030 :       if (isDereferenceable)
    5127             :         SrcMMOFlags |= MachineMemOperand::MODereferenceable;
    5128             : 
    5129        2030 :       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
    5130             :                              DAG.getMemBasePlusOffset(Src, SrcOff, dl),
    5131             :                              SrcPtrInfo.getWithOffset(SrcOff), VT,
    5132        8120 :                              MinAlign(SrcAlign, SrcOff), SrcMMOFlags);
    5133        4060 :       OutChains.push_back(Value.getValue(1));
    5134        2030 :       Store = DAG.getTruncStore(
    5135             :           Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5136        6090 :           DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags);
    5137             :     }
    5138        2326 :     OutChains.push_back(Store);
    5139        2326 :     SrcOff += VTSize;
    5140        2326 :     DstOff += VTSize;
    5141        2326 :     Size -= VTSize;
    5142             :   }
    5143             : 
    5144        3450 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    5145             : }
    5146             : 
    5147          83 : static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
    5148             :                                         SDValue Chain, SDValue Dst, SDValue Src,
    5149             :                                         uint64_t Size, unsigned Align,
    5150             :                                         bool isVol, bool AlwaysInline,
    5151             :                                         MachinePointerInfo DstPtrInfo,
    5152             :                                         MachinePointerInfo SrcPtrInfo) {
    5153             :   // Turn a memmove of undef to nop.
    5154         166 :   if (Src.isUndef())
    5155           0 :     return Chain;
    5156             : 
    5157             :   // Expand memmove to a series of load and store ops if the size operand falls
    5158             :   // below a certain threshold.
    5159          83 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5160         166 :   const DataLayout &DL = DAG.getDataLayout();
    5161          83 :   LLVMContext &C = *DAG.getContext();
    5162          83 :   std::vector<EVT> MemOps;
    5163          83 :   bool DstAlignCanChange = false;
    5164          83 :   MachineFunction &MF = DAG.getMachineFunction();
    5165          83 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    5166          83 :   bool OptSize = shouldLowerMemFuncForSize(MF);
    5167           1 :   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
    5168           1 :   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
    5169             :     DstAlignCanChange = true;
    5170          83 :   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
    5171          83 :   if (Align > SrcAlign)
    5172          70 :     SrcAlign = Align;
    5173         166 :   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
    5174             : 
    5175          83 :   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
    5176             :                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
    5177             :                                 false, false, false, false,
    5178             :                                 DstPtrInfo.getAddrSpace(),
    5179             :                                 SrcPtrInfo.getAddrSpace(),
    5180             :                                 DAG, TLI))
    5181          57 :     return SDValue();
    5182             : 
    5183          26 :   if (DstAlignCanChange) {
    5184           1 :     Type *Ty = MemOps[0].getTypeForEVT(C);
    5185           1 :     unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
    5186           1 :     if (NewAlign > Align) {
    5187             :       // Give the stack frame object a larger alignment if needed.
    5188           2 :       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
    5189           0 :         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
    5190             :       Align = NewAlign;
    5191             :     }
    5192             :   }
    5193             : 
    5194          26 :   MachineMemOperand::Flags MMOFlags =
    5195          26 :       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
    5196          26 :   uint64_t SrcOff = 0, DstOff = 0;
    5197          26 :   SmallVector<SDValue, 8> LoadValues;
    5198          52 :   SmallVector<SDValue, 8> LoadChains;
    5199          52 :   SmallVector<SDValue, 8> OutChains;
    5200          52 :   unsigned NumMemOps = MemOps.size();
    5201          52 :   for (unsigned i = 0; i < NumMemOps; i++) {
    5202          52 :     EVT VT = MemOps[i];
    5203          26 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5204          26 :     SDValue Value;
    5205             : 
    5206             :     bool isDereferenceable =
    5207          52 :       SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
    5208          26 :     MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
    5209          26 :     if (isDereferenceable)
    5210             :       SrcMMOFlags |= MachineMemOperand::MODereferenceable;
    5211             : 
    5212          26 :     Value =
    5213          52 :         DAG.getLoad(VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl),
    5214          78 :                     SrcPtrInfo.getWithOffset(SrcOff), SrcAlign, SrcMMOFlags);
    5215          26 :     LoadValues.push_back(Value);
    5216          52 :     LoadChains.push_back(Value.getValue(1));
    5217          26 :     SrcOff += VTSize;
    5218             :   }
    5219          78 :   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
    5220          26 :   OutChains.clear();
    5221          52 :   for (unsigned i = 0; i < NumMemOps; i++) {
    5222          52 :     EVT VT = MemOps[i];
    5223          26 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5224          26 :     SDValue Store;
    5225             : 
    5226          52 :     Store = DAG.getStore(Chain, dl, LoadValues[i],
    5227             :                          DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5228          78 :                          DstPtrInfo.getWithOffset(DstOff), Align, MMOFlags);
    5229          26 :     OutChains.push_back(Store);
    5230          26 :     DstOff += VTSize;
    5231             :   }
    5232             : 
    5233          78 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    5234             : }
    5235             : 
    5236             : /// \brief Lower the call to 'memset' intrinsic function into a series of store
    5237             : /// operations.
    5238             : ///
    5239             : /// \param DAG Selection DAG where lowered code is placed.
    5240             : /// \param dl Link to corresponding IR location.
    5241             : /// \param Chain Control flow dependency.
    5242             : /// \param Dst Pointer to destination memory location.
    5243             : /// \param Src Value of byte to write into the memory.
    5244             : /// \param Size Number of bytes to write.
    5245             : /// \param Align Alignment of the destination in bytes.
    5246             : /// \param isVol True if destination is volatile.
    5247             : /// \param DstPtrInfo IR information on the memory pointer.
    5248             : /// \returns New head in the control flow, if lowering was successful, empty
    5249             : /// SDValue otherwise.
    5250             : ///
    5251             : /// The function tries to replace 'llvm.memset' intrinsic with several store
    5252             : /// operations and value calculation code. This is usually profitable for small
    5253             : /// memory size.
    5254       21447 : static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
    5255             :                                SDValue Chain, SDValue Dst, SDValue Src,
    5256             :                                uint64_t Size, unsigned Align, bool isVol,
    5257             :                                MachinePointerInfo DstPtrInfo) {
    5258             :   // Turn a memset of undef to nop.
    5259       42894 :   if (Src.isUndef())
    5260           1 :     return Chain;
    5261             : 
    5262             :   // Expand memset to a series of load/store ops if the size operand
    5263             :   // falls below a certain threshold.
    5264       21446 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5265       21446 :   std::vector<EVT> MemOps;
    5266       21446 :   bool DstAlignCanChange = false;
    5267       21446 :   MachineFunction &MF = DAG.getMachineFunction();
    5268       21446 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    5269       21446 :   bool OptSize = shouldLowerMemFuncForSize(MF);
    5270         189 :   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
    5271         189 :   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
    5272             :     DstAlignCanChange = true;
    5273             :   bool IsZeroVal =
    5274       42794 :     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
    5275       42892 :   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
    5276             :                                 Size, (DstAlignCanChange ? 0 : Align), 0,
    5277             :                                 true, IsZeroVal, false, true,
    5278             :                                 DstPtrInfo.getAddrSpace(), ~0u,
    5279             :                                 DAG, TLI))
    5280         941 :     return SDValue();
    5281             : 
    5282       20505 :   if (DstAlignCanChange) {
    5283         125 :     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
    5284         250 :     unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
    5285         125 :     if (NewAlign > Align) {
    5286             :       // Give the stack frame object a larger alignment if needed.
    5287         216 :       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
    5288         103 :         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
    5289             :       Align = NewAlign;
    5290             :     }
    5291             :   }
    5292             : 
    5293       20505 :   SmallVector<SDValue, 8> OutChains;
    5294       20505 :   uint64_t DstOff = 0;
    5295       41010 :   unsigned NumMemOps = MemOps.size();
    5296             : 
    5297             :   // Find the largest store and generate the bit pattern for it.
    5298       20505 :   EVT LargestVT = MemOps[0];
    5299       41274 :   for (unsigned i = 1; i < NumMemOps; i++)
    5300       41538 :     if (MemOps[i].bitsGT(LargestVT))
    5301           0 :       LargestVT = MemOps[i];
    5302       20505 :   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
    5303             : 
    5304       61779 :   for (unsigned i = 0; i < NumMemOps; i++) {
    5305       82548 :     EVT VT = MemOps[i];
    5306       41274 :     unsigned VTSize = VT.getSizeInBits() / 8;
    5307       41274 :     if (VTSize > Size) {
    5308             :       // Issuing an unaligned load / store pair  that overlaps with the previous
    5309             :       // pair. Adjust the offset accordingly.
    5310             :       assert(i == NumMemOps-1 && i != 0);
    5311           3 :       DstOff -= VTSize - Size;
    5312             :     }
    5313             : 
    5314             :     // If this store is smaller than the largest store see whether we can get
    5315             :     // the smaller value for free with a truncate.
    5316       41274 :     SDValue Value = MemSetValue;
    5317       41274 :     if (VT.bitsLT(LargestVT)) {
    5318        1576 :       if (!LargestVT.isVector() && !VT.isVector() &&
    5319          23 :           TLI.isTruncateFree(LargestVT, VT))
    5320          12 :         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
    5321             :       else
    5322        1518 :         Value = getMemsetValue(Src, VT, DAG, dl);
    5323             :     }
    5324             :     assert(Value.getValueType() == VT && "Value with wrong type.");
    5325             :     SDValue Store = DAG.getStore(
    5326             :         Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
    5327             :         DstPtrInfo.getWithOffset(DstOff), Align,
    5328       82548 :         isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone);
    5329       41274 :     OutChains.push_back(Store);
    5330       41274 :     DstOff += VT.getSizeInBits() / 8;
    5331       41274 :     Size -= VTSize;
    5332             :   }
    5333             : 
    5334       61515 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    5335             : }
    5336             : 
    5337        2081 : static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
    5338             :                                             unsigned AS) {
    5339             :   // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
    5340             :   // pointer operands can be losslessly bitcasted to pointers of address space 0
    5341        2081 :   if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) {
    5342           0 :     report_fatal_error("cannot lower memory intrinsic in address space " +
    5343           0 :                        Twine(AS));
    5344             :   }
    5345        2081 : }
    5346             : 
    5347        1700 : SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
    5348             :                                 SDValue Src, SDValue Size, unsigned Align,
    5349             :                                 bool isVol, bool AlwaysInline, bool isTailCall,
    5350             :                                 MachinePointerInfo DstPtrInfo,
    5351             :                                 MachinePointerInfo SrcPtrInfo) {
    5352             :   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
    5353             : 
    5354             :   // Check to see if we should lower the memcpy to loads and stores first.
    5355             :   // For cases within the target-specified limits, this is the best choice.
    5356        1428 :   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
    5357             :   if (ConstantSize) {
    5358             :     // Memcpy with size zero? Just return the original chain.
    5359        1428 :     if (ConstantSize->isNullValue())
    5360        1160 :       return Chain;
    5361             : 
    5362             :     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
    5363             :                                              ConstantSize->getZExtValue(),Align,
    5364        2846 :                                 isVol, false, DstPtrInfo, SrcPtrInfo);
    5365        1423 :     if (Result.getNode())
    5366        1150 :       return Result;
    5367             :   }
    5368             : 
    5369             :   // Then check to see if we should lower the memcpy with target-specific
    5370             :   // code. If the target chooses to do this, this is the next best.
    5371         545 :   if (TSI) {
    5372             :     SDValue Result = TSI->EmitTargetCodeForMemcpy(
    5373             :         *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
    5374         545 :         DstPtrInfo, SrcPtrInfo);
    5375         545 :     if (Result.getNode())
    5376         146 :       return Result;
    5377             :   }
    5378             : 
    5379             :   // If we really need inline code and the target declined to provide it,
    5380             :   // use a (potentially long) sequence of loads and stores.
    5381         399 :   if (AlwaysInline) {
    5382             :     assert(ConstantSize && "AlwaysInline requires a constant size!");
    5383             :     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
    5384             :                                    ConstantSize->getZExtValue(), Align, isVol,
    5385           6 :                                    true, DstPtrInfo, SrcPtrInfo);
    5386             :   }
    5387             : 
    5388         396 :   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
    5389         396 :   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
    5390             : 
    5391             :   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
    5392             :   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
    5393             :   // respect volatile, so they may do things like read or write memory
    5394             :   // beyond the given memory regions. But fixing this isn't easy, and most
    5395             :   // people don't care.
    5396             : 
    5397             :   // Emit a library call.
    5398         396 :   TargetLowering::ArgListTy Args;
    5399         396 :   TargetLowering::ArgListEntry Entry;
    5400         792 :   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
    5401         396 :   Entry.Node = Dst; Args.push_back(Entry);
    5402         396 :   Entry.Node = Src; Args.push_back(Entry);
    5403         396 :   Entry.Node = Size; Args.push_back(Entry);
    5404             :   // FIXME: pass in SDLoc
    5405         792 :   TargetLowering::CallLoweringInfo CLI(*this);
    5406         396 :   CLI.setDebugLoc(dl)
    5407         396 :       .setChain(Chain)
    5408         396 :       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
    5409        1188 :                     Dst.getValueType().getTypeForEVT(*getContext()),
    5410         396 :                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
    5411         396 :                                       TLI->getPointerTy(getDataLayout())),
    5412        4356 :                     std::move(Args))
    5413         396 :       .setDiscardResult()
    5414         792 :       .setTailCall(isTailCall);
    5415             : 
    5416         396 :   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
    5417         396 :   return CallResult.second;
    5418             : }
    5419             : 
    5420         280 : SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
    5421             :                                  SDValue Src, SDValue Size, unsigned Align,
    5422             :                                  bool isVol, bool isTailCall,
    5423             :                                  MachinePointerInfo DstPtrInfo,
    5424             :                                  MachinePointerInfo SrcPtrInfo) {
    5425             :   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
    5426             : 
    5427             :   // Check to see if we should lower the memmove to loads and stores first.
    5428             :   // For cases within the target-specified limits, this is the best choice.
    5429          83 :   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
    5430             :   if (ConstantSize) {
    5431             :     // Memmove with size zero? Just return the original chain.
    5432          83 :     if (ConstantSize->isNullValue())
    5433          26 :       return Chain;
    5434             : 
    5435             :     SDValue Result =
    5436             :       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
    5437             :                                ConstantSize->getZExtValue(), Align, isVol,
    5438         166 :                                false, DstPtrInfo, SrcPtrInfo);
    5439          83 :     if (Result.getNode())
    5440          26 :       return Result;
    5441             :   }
    5442             : 
    5443             :   // Then check to see if we should lower the memmove with target-specific
    5444             :   // code. If the target chooses to do this, this is the next best.
    5445         254 :   if (TSI) {
    5446             :     SDValue Result = TSI->EmitTargetCodeForMemmove(
    5447         254 :         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
    5448         254 :     if (Result.getNode())
    5449          48 :       return Result;
    5450             :   }
    5451             : 
    5452         206 :   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
    5453         206 :   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
    5454             : 
    5455             :   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
    5456             :   // not be safe.  See memcpy above for more details.
    5457             : 
    5458             :   // Emit a library call.
    5459         206 :   TargetLowering::ArgListTy Args;
    5460         206 :   TargetLowering::ArgListEntry Entry;
    5461         412 :   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
    5462         206 :   Entry.Node = Dst; Args.push_back(Entry);
    5463         206 :   Entry.Node = Src; Args.push_back(Entry);
    5464         206 :   Entry.Node = Size; Args.push_back(Entry);
    5465             :   // FIXME:  pass in SDLoc
    5466         412 :   TargetLowering::CallLoweringInfo CLI(*this);
    5467         206 :   CLI.setDebugLoc(dl)
    5468         206 :       .setChain(Chain)
    5469         206 :       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
    5470         618 :                     Dst.getValueType().getTypeForEVT(*getContext()),
    5471         206 :                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
    5472         206 :                                       TLI->getPointerTy(getDataLayout())),
    5473        2266 :                     std::move(Args))
    5474         206 :       .setDiscardResult()
    5475         412 :       .setTailCall(isTailCall);
    5476             : 
    5477         206 :   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
    5478         206 :   return CallResult.second;
    5479             : }
    5480             : 
    5481       21563 : SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
    5482             :                                 SDValue Src, SDValue Size, unsigned Align,
    5483             :                                 bool isVol, bool isTailCall,
    5484             :                                 MachinePointerInfo DstPtrInfo) {
    5485             :   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
    5486             : 
    5487             :   // Check to see if we should lower the memset to stores first.
    5488             :   // For cases within the target-specified limits, this is the best choice.
    5489       21455 :   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
    5490             :   if (ConstantSize) {
    5491             :     // Memset with size zero? Just return the original chain.
    5492       21455 :     if (ConstantSize->isNullValue())
    5493       20522 :       return Chain;
    5494             : 
    5495             :     SDValue Result =
    5496             :       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
    5497       42894 :                       Align, isVol, DstPtrInfo);
    5498             : 
    5499       21447 :     if (Result.getNode())
    5500       20506 :       return Result;
    5501             :   }
    5502             : 
    5503             :   // Then check to see if we should lower the memset with target-specific
    5504             :   // code. If the target chooses to do this, this is the next best.
    5505        1049 :   if (TSI) {
    5506             :     SDValue Result = TSI->EmitTargetCodeForMemset(
    5507        1049 :         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
    5508        1049 :     if (Result.getNode())
    5509         172 :       return Result;
    5510             :   }
    5511             : 
    5512         877 :   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
    5513             : 
    5514             :   // Emit a library call.
    5515        1754 :   Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
    5516         877 :   TargetLowering::ArgListTy Args;
    5517         877 :   TargetLowering::ArgListEntry Entry;
    5518         877 :   Entry.Node = Dst; Entry.Ty = IntPtrTy;
    5519         877 :   Args.push_back(Entry);
    5520         877 :   Entry.Node = Src;
    5521        1754 :   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
    5522         877 :   Args.push_back(Entry);
    5523         877 :   Entry.Node = Size;
    5524         877 :   Entry.Ty = IntPtrTy;
    5525         877 :   Args.push_back(Entry);
    5526             : 
    5527             :   // FIXME: pass in SDLoc
    5528        1754 :   TargetLowering::CallLoweringInfo CLI(*this);
    5529         877 :   CLI.setDebugLoc(dl)
    5530         877 :       .setChain(Chain)
    5531         877 :       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
    5532        2631 :                     Dst.getValueType().getTypeForEVT(*getContext()),
    5533         877 :                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
    5534         877 :                                       TLI->getPointerTy(getDataLayout())),
    5535        9647 :                     std::move(Args))
    5536         877 :       .setDiscardResult()
    5537        1754 :       .setTailCall(isTailCall);
    5538             : 
    5539         877 :   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
    5540         877 :   return CallResult.second;
    5541             : }
    5542             : 
    5543        8603 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5544             :                                 SDVTList VTList, ArrayRef<SDValue> Ops,
    5545             :                                 MachineMemOperand *MMO) {
    5546       17206 :   FoldingSetNodeID ID;
    5547        8603 :   ID.AddInteger(MemVT.getRawBits());
    5548        8603 :   AddNodeIDNode(ID, Opcode, VTList, Ops);
    5549       17206 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    5550        8603 :   void* IP = nullptr;
    5551        8603 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    5552           0 :     cast<AtomicSDNode>(E)->refineAlignment(MMO);
    5553           0 :     return SDValue(E, 0);
    5554             :   }
    5555             : 
    5556        8603 :   auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
    5557        8603 :                                     VTList, MemVT, MMO);
    5558        8603 :   createOperands(N, Ops);
    5559             : 
    5560       17206 :   CSEMap.InsertNode(N, IP);
    5561        8603 :   InsertNode(N);
    5562        8603 :   return SDValue(N, 0);
    5563             : }
    5564             : 
    5565        1066 : SDValue SelectionDAG::getAtomicCmpSwap(
    5566             :     unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain,
    5567             :     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
    5568             :     unsigned Alignment, AtomicOrdering SuccessOrdering,
    5569             :     AtomicOrdering FailureOrdering, SyncScope::ID SSID) {
    5570             :   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
    5571             :          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
    5572             :   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
    5573             : 
    5574        1066 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5575        1066 :     Alignment = getEVTAlignment(MemVT);
    5576             : 
    5577        1066 :   MachineFunction &MF = getMachineFunction();
    5578             : 
    5579             :   // FIXME: Volatile isn't really correct; we should keep track of atomic
    5580             :   // orderings in the memoperand.
    5581        1066 :   auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad |
    5582        1066 :                MachineMemOperand::MOStore;
    5583             :   MachineMemOperand *MMO =
    5584        2132 :     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
    5585        2132 :                             AAMDNodes(), nullptr, SSID, SuccessOrdering,
    5586        1066 :                             FailureOrdering);
    5587             : 
    5588        1066 :   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO);
    5589             : }
    5590             : 
    5591        2612 : SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
    5592             :                                        EVT MemVT, SDVTList VTs, SDValue Chain,
    5593             :                                        SDValue Ptr, SDValue Cmp, SDValue Swp,
    5594             :                                        MachineMemOperand *MMO) {
    5595             :   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
    5596             :          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
    5597             :   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
    5598             : 
    5599        2612 :   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
    5600        2612 :   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
    5601             : }
    5602             : 
    5603        4268 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5604             :                                 SDValue Chain, SDValue Ptr, SDValue Val,
    5605             :                                 const Value *PtrVal, unsigned Alignment,
    5606             :                                 AtomicOrdering Ordering,
    5607             :                                 SyncScope::ID SSID) {
    5608        4268 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5609        3729 :     Alignment = getEVTAlignment(MemVT);
    5610             : 
    5611        4268 :   MachineFunction &MF = getMachineFunction();
    5612             :   // An atomic store does not load. An atomic load does not store.
    5613             :   // (An atomicrmw obviously both loads and stores.)
    5614             :   // For now, atomics are considered to be volatile always, and they are
    5615             :   // chained as such.
    5616             :   // FIXME: Volatile isn't really correct; we should keep track of atomic
    5617             :   // orderings in the memoperand.
    5618        4268 :   auto Flags = MachineMemOperand::MOVolatile;
    5619        4268 :   if (Opcode != ISD::ATOMIC_STORE)
    5620             :     Flags |= MachineMemOperand::MOLoad;
    5621        4268 :   if (Opcode != ISD::ATOMIC_LOAD)
    5622             :     Flags |= MachineMemOperand::MOStore;
    5623             : 
    5624             :   MachineMemOperand *MMO =
    5625       17072 :     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
    5626       12804 :                             MemVT.getStoreSize(), Alignment, AAMDNodes(),
    5627        4268 :                             nullptr, SSID, Ordering);
    5628             : 
    5629        4268 :   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
    5630             : }
    5631             : 
    5632        5369 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5633             :                                 SDValue Chain, SDValue Ptr, SDValue Val,
    5634             :                                 MachineMemOperand *MMO) {
    5635             :   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
    5636             :           Opcode == ISD::ATOMIC_LOAD_SUB ||
    5637             :           Opcode == ISD::ATOMIC_LOAD_AND ||
    5638             :           Opcode == ISD::ATOMIC_LOAD_OR ||
    5639             :           Opcode == ISD::ATOMIC_LOAD_XOR ||
    5640             :           Opcode == ISD::ATOMIC_LOAD_NAND ||
    5641             :           Opcode == ISD::ATOMIC_LOAD_MIN ||
    5642             :           Opcode == ISD::ATOMIC_LOAD_MAX ||
    5643             :           Opcode == ISD::ATOMIC_LOAD_UMIN ||
    5644             :           Opcode == ISD::ATOMIC_LOAD_UMAX ||
    5645             :           Opcode == ISD::ATOMIC_SWAP ||
    5646             :           Opcode == ISD::ATOMIC_STORE) &&
    5647             :          "Invalid Atomic Op");
    5648             : 
    5649       10738 :   EVT VT = Val.getValueType();
    5650             : 
    5651        6011 :   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
    5652       10738 :                                                getVTList(VT, MVT::Other);
    5653        5369 :   SDValue Ops[] = {Chain, Ptr, Val};
    5654        5369 :   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
    5655             : }
    5656             : 
    5657         622 : SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
    5658             :                                 EVT VT, SDValue Chain, SDValue Ptr,
    5659             :                                 MachineMemOperand *MMO) {
    5660             :   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
    5661             : 
    5662         622 :   SDVTList VTs = getVTList(VT, MVT::Other);
    5663         622 :   SDValue Ops[] = {Chain, Ptr};
    5664         622 :   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
    5665             : }
    5666             : 
    5667             : /// getMergeValues - Create a MERGE_VALUES node from the given operands.
    5668      278429 : SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
    5669      278429 :   if (Ops.size() == 1)
    5670      229932 :     return Ops[0];
    5671             : 
    5672       48497 :   SmallVector<EVT, 4> VTs;
    5673       48497 :   VTs.reserve(Ops.size());
    5674      147382 :   for (unsigned i = 0; i < Ops.size(); ++i)
    5675      296655 :     VTs.push_back(Ops[i].getValueType());
    5676       48497 :   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
    5677             : }
    5678             : 
    5679        6532 : SDValue SelectionDAG::getMemIntrinsicNode(
    5680             :     unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
    5681             :     EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol,
    5682             :     bool ReadMem, bool WriteMem, unsigned Size) {
    5683        6532 :   if (Align == 0)  // Ensure that codegen never sees alignment 0
    5684        3042 :     Align = getEVTAlignment(MemVT);
    5685             : 
    5686        6532 :   MachineFunction &MF = getMachineFunction();
    5687        6532 :   auto Flags = MachineMemOperand::MONone;
    5688        6532 :   if (WriteMem)
    5689             :     Flags |= MachineMemOperand::MOStore;
    5690        6532 :   if (ReadMem)
    5691             :     Flags |= MachineMemOperand::MOLoad;
    5692        6532 :   if (Vol)
    5693             :     Flags |= MachineMemOperand::MOVolatile;
    5694        6532 :   if (!Size)
    5695        6490 :     Size = MemVT.getStoreSize();
    5696             :   MachineMemOperand *MMO =
    5697        6532 :     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
    5698             : 
    5699        6532 :   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
    5700             : }
    5701             : 
    5702       12503 : SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
    5703             :                                           SDVTList VTList,
    5704             :                                           ArrayRef<SDValue> Ops, EVT MemVT,
    5705             :                                           MachineMemOperand *MMO) {
    5706             :   assert((Opcode == ISD::INTRINSIC_VOID ||
    5707             :           Opcode == ISD::INTRINSIC_W_CHAIN ||
    5708             :           Opcode == ISD::PREFETCH ||
    5709             :           Opcode == ISD::LIFETIME_START ||
    5710             :           Opcode == ISD::LIFETIME_END ||
    5711             :           ((int)Opcode <= std::numeric_limits<int>::max() &&
    5712             :            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
    5713             :          "Opcode is not a memory-accessing opcode!");
    5714             : 
    5715             :   // Memoize the node unless it returns a flag.
    5716             :   MemIntrinsicSDNode *N;
    5717       12503 :   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
    5718       23113 :     FoldingSetNodeID ID;
    5719       11559 :     AddNodeIDNode(ID, Opcode, VTList, Ops);
    5720       23118 :     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    5721       11559 :     void *IP = nullptr;
    5722       11559 :     if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    5723          10 :       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
    5724           5 :       return SDValue(E, 0);
    5725             :     }
    5726             : 
    5727       23108 :     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
    5728             :                                       VTList, MemVT, MMO);
    5729       11554 :     createOperands(N, Ops);
    5730             : 
    5731       23108 :   CSEMap.InsertNode(N, IP);
    5732             :   } else {
    5733        1888 :     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
    5734             :                                       VTList, MemVT, MMO);
    5735         944 :     createOperands(N, Ops);
    5736             :   }
    5737       12498 :   InsertNode(N);
    5738       12498 :   return SDValue(N, 0);
    5739             : }
    5740             : 
    5741             : /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
    5742             : /// MachinePointerInfo record from it.  This is particularly useful because the
    5743             : /// code generator has many cases where it doesn't bother passing in a
    5744             : /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
    5745       38755 : static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
    5746             :                                            int64_t Offset = 0) {
    5747             :   // If this is FI+Offset, we can model it.
    5748       12729 :   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
    5749             :     return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
    5750       12729 :                                              FI->getIndex(), Offset);
    5751             : 
    5752             :   // If this is (FI+Offset1)+Offset2, we can model it.
    5753       26026 :   if (Ptr.getOpcode() != ISD::ADD ||
    5754       40864 :       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
    5755       27640 :       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
    5756             :     return MachinePointerInfo();
    5757             : 
    5758        8208 :   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
    5759             :   return MachinePointerInfo::getFixedStack(
    5760             :       DAG.getMachineFunction(), FI,
    5761       10944 :       Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
    5762             : }
    5763             : 
    5764             : /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
    5765             : /// MachinePointerInfo record from it.  This is particularly useful because the
    5766             : /// code generator has many cases where it doesn't bother passing in a
    5767             : /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
    5768       17689 : static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
    5769             :                                            SDValue OffsetOp) {
    5770             :   // If the 'Offset' value isn't a constant, we can't handle this.
    5771           5 :   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
    5772           5 :     return InferPointerInfo(DAG, Ptr, OffsetNode->getSExtValue());
    5773       35368 :   if (OffsetOp.isUndef())
    5774       17684 :     return InferPointerInfo(DAG, Ptr);
    5775             :   return MachinePointerInfo();
    5776             : }
    5777             : 
    5778      585445 : SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
    5779             :                               EVT VT, const SDLoc &dl, SDValue Chain,
    5780             :                               SDValue Ptr, SDValue Offset,
    5781             :                               MachinePointerInfo PtrInfo, EVT MemVT,
    5782             :                               unsigned Alignment,
    5783             :                               MachineMemOperand::Flags MMOFlags,
    5784             :                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
    5785             :   assert(Chain.getValueType() == MVT::Other &&
    5786             :         "Invalid chain type");
    5787      585445 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5788       95642 :     Alignment = getEVTAlignment(MemVT);
    5789             : 
    5790      585445 :   MMOFlags |= MachineMemOperand::MOLoad;
    5791             :   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
    5792             :   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
    5793             :   // clients.
    5794      585445 :   if (PtrInfo.V.isNull())
    5795       17689 :     PtrInfo = InferPointerInfo(*this, Ptr, Offset);
    5796             : 
    5797      585445 :   MachineFunction &MF = getMachineFunction();
    5798      585445 :   MachineMemOperand *MMO = MF.getMachineMemOperand(
    5799     1170890 :       PtrInfo, MMOFlags, MemVT.getStoreSize(), Alignment, AAInfo, Ranges);
    5800      585445 :   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
    5801             : }
    5802             : 
    5803      789215 : SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
    5804             :                               EVT VT, const SDLoc &dl, SDValue Chain,
    5805             :                               SDValue Ptr, SDValue Offset, EVT MemVT,
    5806             :                               MachineMemOperand *MMO) {
    5807      791895 :   if (VT == MemVT) {
    5808             :     ExtType = ISD::NON_EXTLOAD;
    5809             :   } else if (ExtType == ISD::NON_EXTLOAD) {
    5810             :     assert(VT == MemVT && "Non-extending load from different memory type!");
    5811             :   } else {
    5812             :     // Extending load.
    5813             :     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
    5814             :            "Should only be an extending load, not truncating!");
    5815             :     assert(VT.isInteger() == MemVT.isInteger() &&
    5816             :            "Cannot convert from FP to Int or Int -> FP!");
    5817             :     assert(VT.isVector() == MemVT.isVector() &&
    5818             :            "Cannot use an ext load to convert to or from a vector!");
    5819             :     assert((!VT.isVector() ||
    5820             :             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
    5821             :            "Cannot use an ext load to change the number of vector elements!");
    5822             :   }
    5823             : 
    5824      789215 :   bool Indexed = AM != ISD::UNINDEXED;
    5825             :   assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
    5826             : 
    5827             :   SDVTList VTs = Indexed ?
    5828     1578826 :     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
    5829      789215 :   SDValue Ops[] = { Chain, Ptr, Offset };
    5830     1578430 :   FoldingSetNodeID ID;
    5831      789215 :   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
    5832      789215 :   ID.AddInteger(MemVT.getRawBits());
    5833     1578430 :   ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
    5834             :       dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
    5835     1578430 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    5836      789215 :   void *IP = nullptr;
    5837      789215 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    5838       34996 :     cast<LoadSDNode>(E)->refineAlignment(MMO);
    5839       17498 :     return SDValue(E, 0);
    5840             :   }
    5841     1543434 :   auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
    5842      771717 :                                   ExtType, MemVT, MMO);
    5843      771717 :   createOperands(N, Ops);
    5844             : 
    5845     1543434 :   CSEMap.InsertNode(N, IP);
    5846      771717 :   InsertNode(N);
    5847      771717 :   return SDValue(N, 0);
    5848             : }
    5849             : 
    5850      531000 : SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
    5851             :                               SDValue Ptr, MachinePointerInfo PtrInfo,
    5852             :                               unsigned Alignment,
    5853             :                               MachineMemOperand::Flags MMOFlags,
    5854             :                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
    5855     1062000 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    5856             :   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
    5857      531000 :                  PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
    5858             : }
    5859             : 
    5860      180266 : SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
    5861             :                               SDValue Ptr, MachineMemOperand *MMO) {
    5862      360532 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    5863             :   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
    5864      180266 :                  VT, MMO);
    5865             : }
    5866             : 
    5867       37541 : SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
    5868             :                                  EVT VT, SDValue Chain, SDValue Ptr,
    5869             :                                  MachinePointerInfo PtrInfo, EVT MemVT,
    5870             :                                  unsigned Alignment,
    5871             :                                  MachineMemOperand::Flags MMOFlags,
    5872             :                                  const AAMDNodes &AAInfo) {
    5873       75082 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    5874             :   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
    5875       37541 :                  MemVT, Alignment, MMOFlags, AAInfo);
    5876             : }
    5877             : 
    5878       22312 : SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
    5879             :                                  EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
    5880             :                                  MachineMemOperand *MMO) {
    5881       44624 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    5882             :   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
    5883       22312 :                  MemVT, MMO);
    5884             : }
    5885             : 
    5886         396 : SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
    5887             :                                      SDValue Base, SDValue Offset,
    5888             :                                      ISD::MemIndexedMode AM) {
    5889         396 :   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
    5890             :   assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
    5891             :   // Don't propagate the invariant or dereferenceable flags.
    5892             :   auto MMOFlags =
    5893         396 :       LD->getMemOperand()->getFlags() &
    5894        1188 :       ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
    5895             :   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
    5896        1188 :                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
    5897             :                  LD->getMemoryVT(), LD->getAlignment(), MMOFlags,
    5898        2376 :                  LD->getAAInfo());
    5899             : }
    5900             : 
    5901      688786 : SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    5902             :                                SDValue Ptr, MachinePointerInfo PtrInfo,
    5903             :                                unsigned Alignment,
    5904             :                                MachineMemOperand::Flags MMOFlags,
    5905             :                                const AAMDNodes &AAInfo) {
    5906             :   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
    5907      688786 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5908      369786 :     Alignment = getEVTAlignment(Val.getValueType());
    5909             : 
    5910      688786 :   MMOFlags |= MachineMemOperand::MOStore;
    5911             :   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
    5912             : 
    5913      688786 :   if (PtrInfo.V.isNull())
    5914       16477 :     PtrInfo = InferPointerInfo(*this, Ptr);
    5915             : 
    5916      688786 :   MachineFunction &MF = getMachineFunction();
    5917      688786 :   MachineMemOperand *MMO = MF.getMachineMemOperand(
    5918     2755144 :       PtrInfo, MMOFlags, Val.getValueType().getStoreSize(), Alignment, AAInfo);
    5919      688786 :   return getStore(Chain, dl, Val, Ptr, MMO);
    5920             : }
    5921             : 
    5922     1004495 : SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    5923             :                                SDValue Ptr, MachineMemOperand *MMO) {
    5924             :   assert(Chain.getValueType() == MVT::Other &&
    5925             :         "Invalid chain type");
    5926     2008990 :   EVT VT = Val.getValueType();
    5927     1004495 :   SDVTList VTs = getVTList(MVT::Other);
    5928     2008990 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    5929     1004495 :   SDValue Ops[] = { Chain, Val, Ptr, Undef };
    5930     2008990 :   FoldingSetNodeID ID;
    5931     1004495 :   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
    5932     1004495 :   ID.AddInteger(VT.getRawBits());
    5933     2008990 :   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
    5934             :       dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
    5935     2008990 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    5936     1004495 :   void *IP = nullptr;
    5937     1004495 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    5938       10778 :     cast<StoreSDNode>(E)->refineAlignment(MMO);
    5939        5389 :     return SDValue(E, 0);
    5940             :   }
    5941     2997318 :   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    5942      999106 :                                    ISD::UNINDEXED, false, VT, MMO);
    5943      999106 :   createOperands(N, Ops);
    5944             : 
    5945     1998212 :   CSEMap.InsertNode(N, IP);
    5946      999106 :   InsertNode(N);
    5947      999106 :   return SDValue(N, 0);
    5948             : }
    5949             : 
    5950       39340 : SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    5951             :                                     SDValue Ptr, MachinePointerInfo PtrInfo,
    5952             :                                     EVT SVT, unsigned Alignment,
    5953             :                                     MachineMemOperand::Flags MMOFlags,
    5954             :                                     const AAMDNodes &AAInfo) {
    5955             :   assert(Chain.getValueType() == MVT::Other &&
    5956             :         "Invalid chain type");
    5957       39340 :   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
    5958        1030 :     Alignment = getEVTAlignment(SVT);
    5959             : 
    5960       39340 :   MMOFlags |= MachineMemOperand::MOStore;
    5961             :   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
    5962             : 
    5963       39340 :   if (PtrInfo.V.isNull())
    5964        4589 :     PtrInfo = InferPointerInfo(*this, Ptr);
    5965             : 
    5966       39340 :   MachineFunction &MF = getMachineFunction();
    5967       39340 :   MachineMemOperand *MMO = MF.getMachineMemOperand(
    5968       78680 :       PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
    5969       39340 :   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
    5970             : }
    5971             : 
    5972       51306 : SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
    5973             :                                     SDValue Ptr, EVT SVT,
    5974             :                                     MachineMemOperand *MMO) {
    5975      102612 :   EVT VT = Val.getValueType();
    5976             : 
    5977             :   assert(Chain.getValueType() == MVT::Other &&
    5978             :         "Invalid chain type");
    5979       51834 :   if (VT == SVT)
    5980       27787 :     return getStore(Chain, dl, Val, Ptr, MMO);
    5981             : 
    5982             :   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
    5983             :          "Should only be a truncating store, not extending!");
    5984             :   assert(VT.isInteger() == SVT.isInteger() &&
    5985             :          "Can't do FP-INT conversion!");
    5986             :   assert(VT.isVector() == SVT.isVector() &&
    5987             :          "Cannot use trunc store to convert to or from a vector!");
    5988             :   assert((!VT.isVector() ||
    5989             :           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
    5990             :          "Cannot use trunc store to change the number of vector elements!");
    5991             : 
    5992       23519 :   SDVTList VTs = getVTList(MVT::Other);
    5993       47038 :   SDValue Undef = getUNDEF(Ptr.getValueType());
    5994       23519 :   SDValue Ops[] = { Chain, Val, Ptr, Undef };
    5995       23519 :   FoldingSetNodeID ID;
    5996       23519 :   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
    5997       23519 :   ID.AddInteger(SVT.getRawBits());
    5998       47038 :   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
    5999             :       dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
    6000       47038 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6001       23519 :   void *IP = nullptr;
    6002       23519 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6003        2342 :     cast<StoreSDNode>(E)->refineAlignment(MMO);
    6004        1171 :     return SDValue(E, 0);
    6005             :   }
    6006       67044 :   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    6007       22348 :                                    ISD::UNINDEXED, true, SVT, MMO);
    6008       22348 :   createOperands(N, Ops);
    6009             : 
    6010       44696 :   CSEMap.InsertNode(N, IP);
    6011       22348 :   InsertNode(N);
    6012       22348 :   return SDValue(N, 0);
    6013             : }
    6014             : 
    6015         234 : SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
    6016             :                                       SDValue Base, SDValue Offset,
    6017             :                                       ISD::MemIndexedMode AM) {
    6018         234 :   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
    6019             :   assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
    6020         468 :   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
    6021         702 :   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
    6022         468 :   FoldingSetNodeID ID;
    6023         234 :   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
    6024         468 :   ID.AddInteger(ST->getMemoryVT().getRawBits());
    6025         468 :   ID.AddInteger(ST->getRawSubclassData());
    6026         468 :   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
    6027         234 :   void *IP = nullptr;
    6028         234 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
    6029           0 :     return SDValue(E, 0);
    6030             : 
    6031         234 :   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
    6032         702 :                                    ST->isTruncatingStore(), ST->getMemoryVT(),
    6033         702 :                                    ST->getMemOperand());
    6034         234 :   createOperands(N, Ops);
    6035             : 
    6036         468 :   CSEMap.InsertNode(N, IP);
    6037         234 :   InsertNode(N);
    6038         234 :   return SDValue(N, 0);
    6039             : }
    6040             : 
    6041         316 : SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
    6042             :                                     SDValue Ptr, SDValue Mask, SDValue Src0,
    6043             :                                     EVT MemVT, MachineMemOperand *MMO,
    6044             :                                     ISD::LoadExtType ExtTy, bool isExpanding) {
    6045         316 :   SDVTList VTs = getVTList(VT, MVT::Other);
    6046         316 :   SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
    6047         632 :   FoldingSetNodeID ID;
    6048         316 :   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
    6049         316 :   ID.AddInteger(VT.getRawBits());
    6050         632 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
    6051             :       dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO));
    6052         632 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6053         316 :   void *IP = nullptr;
    6054         316 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6055           0 :     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
    6056           0 :     return SDValue(E, 0);
    6057             :   }
    6058         632 :   auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    6059         316 :                                         ExtTy, isExpanding, MemVT, MMO);
    6060         316 :   createOperands(N, Ops);
    6061             : 
    6062         632 :   CSEMap.InsertNode(N, IP);
    6063         316 :   InsertNode(N);
    6064         316 :   return SDValue(N, 0);
    6065             : }
    6066             : 
    6067         199 : SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
    6068             :                                      SDValue Val, SDValue Ptr, SDValue Mask,
    6069             :                                      EVT MemVT, MachineMemOperand *MMO,
    6070             :                                      bool IsTruncating, bool IsCompressing) {
    6071             :   assert(Chain.getValueType() == MVT::Other &&
    6072             :         "Invalid chain type");
    6073         398 :   EVT VT = Val.getValueType();
    6074         199 :   SDVTList VTs = getVTList(MVT::Other);
    6075         199 :   SDValue Ops[] = { Chain, Ptr, Mask, Val };
    6076         398 :   FoldingSetNodeID ID;
    6077         199 :   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
    6078         199 :   ID.AddInteger(VT.getRawBits());
    6079         398 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
    6080             :       dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO));
    6081         398 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6082         199 :   void *IP = nullptr;
    6083         199 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6084           0 :     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
    6085           0 :     return SDValue(E, 0);
    6086             :   }
    6087         398 :   auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
    6088         199 :                                          IsTruncating, IsCompressing, MemVT, MMO);
    6089         199 :   createOperands(N, Ops);
    6090             : 
    6091         398 :   CSEMap.InsertNode(N, IP);
    6092         199 :   InsertNode(N);
    6093         199 :   return SDValue(N, 0);
    6094             : }
    6095             : 
    6096         290 : SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
    6097             :                                       ArrayRef<SDValue> Ops,
    6098             :                                       MachineMemOperand *MMO) {
    6099             :   assert(Ops.size() == 5 && "Incompatible number of operands");
    6100             : 
    6101         580 :   FoldingSetNodeID ID;
    6102         290 :   AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
    6103         290 :   ID.AddInteger(VT.getRawBits());
    6104         580 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
    6105             :       dl.getIROrder(), VTs, VT, MMO));
    6106         580 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6107         290 :   void *IP = nullptr;
    6108         290 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6109           0 :     cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
    6110           0 :     return SDValue(E, 0);
    6111             :   }
    6112             : 
    6113         290 :   auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
    6114         290 :                                           VTs, VT, MMO);
    6115         290 :   createOperands(N, Ops);
    6116             : 
    6117             :   assert(N->getValue().getValueType() == N->getValueType(0) &&
    6118             :          "Incompatible type of the PassThru value in MaskedGatherSDNode");
    6119             :   assert(N->getMask().getValueType().getVectorNumElements() ==
    6120             :              N->getValueType(0).getVectorNumElements() &&
    6121             :          "Vector width mismatch between mask and data");
    6122             :   assert(N->getIndex().getValueType().getVectorNumElements() ==
    6123             :              N->getValueType(0).getVectorNumElements() &&
    6124             :          "Vector width mismatch between index and data");
    6125             : 
    6126         580 :   CSEMap.InsertNode(N, IP);
    6127         290 :   InsertNode(N);
    6128         290 :   return SDValue(N, 0);
    6129             : }
    6130             : 
    6131         178 : SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
    6132             :                                        ArrayRef<SDValue> Ops,
    6133             :                                        MachineMemOperand *MMO) {
    6134             :   assert(Ops.size() == 5 && "Incompatible number of operands");
    6135             : 
    6136         356 :   FoldingSetNodeID ID;
    6137         178 :   AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
    6138         178 :   ID.AddInteger(VT.getRawBits());
    6139         356 :   ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
    6140             :       dl.getIROrder(), VTs, VT, MMO));
    6141         356 :   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
    6142         178 :   void *IP = nullptr;
    6143         178 :   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
    6144           0 :     cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
    6145           0 :     return SDValue(E, 0);
    6146             :   }
    6147         178 :   auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
    6148         178 :                                            VTs, VT, MMO);
    6149         178 :   createOperands(N, Ops);
    6150             : 
    6151             :   assert(N->getMask().getValueType().getVectorNumElements() ==
    6152             :              N->getValue().getValueType().getVectorNumElements() &&
    6153             :          "Vector width mismatch between mask and data");
    6154             :   assert(N->getIndex().getValueType().getVectorNumElements() ==
    6155             :              N->getValue().getValueType().getVectorNumElements() &&
    6156             :          "Vector width mismatch between index and data");
    6157             : 
    6158         356 :   CSEMap.InsertNode(N, IP);
    6159         178 :   InsertNode(N);
    6160         178 :   return SDValue(N, 0);
    6161             : }
    6162             : 
    6163         245 : SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
    6164             :                                SDValue Ptr, SDValue SV, unsigned Align) {
    6165         735 :   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
    6166         490 :   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
    6167             : }
    6168             : 
    6169        4149 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    6170             :                               ArrayRef<SDUse> Ops) {
    6171        4149 :   switch (Ops.size()) {
    6172           0 :   case 0: return getNode(Opcode, DL, VT);
    6173         138 :   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
    6174        7840 :   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
    6175         160 :   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
    6176             :   default: break;
    6177             :   }
    6178             : 
    6179             :   // Copy from an SDUse array into an SDValue array for use with
    6180             :   // the regular getNode logic.
    6181        4206 :   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
    6182        4206 :   return getNode(Opcode, DL, VT, NewOps);
    6183             : }
    6184             : 
    6185     3198681 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
    6186             :                               ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
    6187     3198681 :   unsigned NumOps = Ops.size();
    6188     3198681 :   switch (NumOps) {
    6189           0 :   case 0: return getNode(Opcode, DL, VT);
    6190     1170931 :   case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
    6191     1865156 :   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
    6192      900588 :   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
    6193             :   default: break;
    6194             :   }
    6195             : 
    6196      794976 :   switch (Opcode) {
    6197             :   default: break;
    6198        6442 :   case ISD::CONCAT_VECTORS:
    6199             :     // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
    6200        6442 :     if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
    6201         233 :       return V;
    6202        6209 :     break;
    6203             :   case ISD::SELECT_CC:
    6204             :     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
    6205             :     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
    6206             :            "LHS and RHS of condition must have same type!");
    6207             :     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
    6208             :            "True and False arms of SelectCC must have same type!");
    6209             :     assert(Ops[2].getValueType() == VT &&
    6210             :            "select_cc node must be of same type as true and false value!");
    6211             :     break;
    6212             :   case ISD::BR_CC:
    6213             :     assert(NumOps == 5 && "BR_CC takes 5 operands!");
    6214             :     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
    6215             :            "LHS/RHS of comparison should match types!");
    6216             :     break;
    6217             :   }
    6218             : 
    6219             :   // Memoize nodes.
    6220             :   SDNode *N;
    6221      794743 :   SDVTList VTs = getVTList(VT);
    6222             : 
    6223      794743 :   if (VT != MVT::Glue) {
    6224     1337010 :     FoldingSetNodeID ID;
    6225      794743 :     AddNodeIDNode(ID, Opcode, VTs, Ops);
    6226      794743 :     void *IP = nullptr;
    6227             : 
    6228      794743 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    6229      252476 :       return SDValue(E, 0);
    6230             : 
    6231     1084534 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    6232      542267 :     createOperands(N, Ops);
    6233             : 
    6234     1084534 :     CSEMap.InsertNode(N, IP);
    6235             :   } else {
    6236           0 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    6237           0 :     createOperands(N, Ops);
    6238             :   }
    6239             : 
    6240      542267 :   InsertNode(N);
    6241      542267 :   return SDValue(N, 0);
    6242             : }
    6243             : 
    6244       10395 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
    6245             :                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
    6246       10395 :   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
    6247             : }
    6248             : 
    6249     2870920 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6250             :                               ArrayRef<SDValue> Ops) {
    6251     2870920 :   if (VTList.NumVTs == 1)
    6252      532804 :     return getNode(Opcode, DL, VTList.VTs[0], Ops);
    6253             : 
    6254             : #if 0
    6255             :   switch (Opcode) {
    6256             :   // FIXME: figure out how to safely handle things like
    6257             :   // int foo(int x) { return 1 << (x & 255); }
    6258             :   // int bar() { return foo(256); }
    6259             :   case ISD::SRA_PARTS:
    6260             :   case ISD::SRL_PARTS:
    6261             :   case ISD::SHL_PARTS:
    6262             :     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
    6263             :         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
    6264             :       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
    6265             :     else if (N3.getOpcode() == ISD::AND)
    6266             :       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
    6267             :         // If the and is only masking out bits that cannot effect the shift,
    6268             :         // eliminate the and.
    6269             :         unsigned NumBits = VT.getScalarSizeInBits()*2;
    6270             :         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
    6271             :           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
    6272             :       }
    6273             :     break;
    6274             :   }
    6275             : #endif
    6276             : 
    6277             :   // Memoize the node unless it returns a flag.
    6278             :   SDNode *N;
    6279     2338116 :   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
    6280     2170956 :     FoldingSetNodeID ID;
    6281     1112499 :     AddNodeIDNode(ID, Opcode, VTList, Ops);
    6282     1112499 :     void *IP = nullptr;
    6283     1112499 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
    6284       54042 :       return SDValue(E, 0);
    6285             : 
    6286     2116914 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
    6287     1058457 :     createOperands(N, Ops);
    6288     2116914 :     CSEMap.InsertNode(N, IP);
    6289             :   } else {
    6290     2451234 :     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
    6291     1225617 :     createOperands(N, Ops);
    6292             :   }
    6293     2284074 :   InsertNode(N);
    6294     2284074 :   return SDValue(N, 0);
    6295             : }
    6296             : 
    6297           0 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
    6298             :                               SDVTList VTList) {
    6299           0 :   return getNode(Opcode, DL, VTList, None);
    6300             : }
    6301             : 
    6302        3395 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6303             :                               SDValue N1) {
    6304        3395 :   SDValue Ops[] = { N1 };
    6305        3395 :   return getNode(Opcode, DL, VTList, Ops);
    6306             : }
    6307             : 
    6308      283519 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6309             :                               SDValue N1, SDValue N2) {
    6310      283519 :   SDValue Ops[] = { N1, N2 };
    6311      283519 :   return getNode(Opcode, DL, VTList, Ops);
    6312             : }
    6313             : 
    6314      130321 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6315             :                               SDValue N1, SDValue N2, SDValue N3) {
    6316      130321 :   SDValue Ops[] = { N1, N2, N3 };
    6317      130321 :   return getNode(Opcode, DL, VTList, Ops);
    6318             : }
    6319             : 
    6320         745 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6321             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
    6322         745 :   SDValue Ops[] = { N1, N2, N3, N4 };
    6323         745 :   return getNode(Opcode, DL, VTList, Ops);
    6324             : }
    6325             : 
    6326         329 : SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
    6327             :                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
    6328             :                               SDValue N5) {
    6329         329 :   SDValue Ops[] = { N1, N2, N3, N4, N5 };
    6330         329 :   return getNode(Opcode, DL, VTList, Ops);
    6331             : }
    6332             : 
    6333    28761278 : SDVTList SelectionDAG::getVTList(EVT VT) {
    6334    57522556 :   return makeVTList(SDNode::getValueTypeList(VT), 1);
    6335             : }
    6336             : 
    6337     3611910 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
    6338     7223820 :   FoldingSetNodeID ID;
    6339     3611910 :   ID.AddInteger(2U);
    6340     3611910 :   ID.AddInteger(VT1.getRawBits());
    6341     3611910 :   ID.AddInteger(VT2.getRawBits());
    6342             : 
    6343     3611910 :   void *IP = nullptr;
    6344     7223820 :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6345     3611910 :   if (!Result) {
    6346      129826 :     EVT *Array = Allocator.Allocate<EVT>(2);
    6347       64913 :     Array[0] = VT1;
    6348       64913 :     Array[1] = VT2;
    6349      194739 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
    6350       64913 :     VTListMap.InsertNode(Result, IP);
    6351             :   }
    6352    10835730 :   return Result->getSDVTList();
    6353             : }
    6354             : 
    6355       39007 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
    6356       78014 :   FoldingSetNodeID ID;
    6357       39007 :   ID.AddInteger(3U);
    6358       39007 :   ID.AddInteger(VT1.getRawBits());
    6359       39007 :   ID.AddInteger(VT2.getRawBits());
    6360       39007 :   ID.AddInteger(VT3.getRawBits());
    6361             : 
    6362       39007 :   void *IP = nullptr;
    6363       78014 :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6364       39007 :   if (!Result) {
    6365        8494 :     EVT *Array = Allocator.Allocate<EVT>(3);
    6366        4247 :     Array[0] = VT1;
    6367        4247 :     Array[1] = VT2;
    6368        4247 :     Array[2] = VT3;
    6369       12741 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
    6370        4247 :     VTListMap.InsertNode(Result, IP);
    6371             :   }
    6372      117021 :   return Result->getSDVTList();
    6373             : }
    6374             : 
    6375          64 : SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
    6376         128 :   FoldingSetNodeID ID;
    6377          64 :   ID.AddInteger(4U);
    6378          64 :   ID.AddInteger(VT1.getRawBits());
    6379          64 :   ID.AddInteger(VT2.getRawBits());
    6380          64 :   ID.AddInteger(VT3.getRawBits());
    6381          64 :   ID.AddInteger(VT4.getRawBits());
    6382             : 
    6383          64 :   void *IP = nullptr;
    6384         128 :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6385          64 :   if (!Result) {
    6386          40 :     EVT *Array = Allocator.Allocate<EVT>(4);
    6387          20 :     Array[0] = VT1;
    6388          20 :     Array[1] = VT2;
    6389          20 :     Array[2] = VT3;
    6390          20 :     Array[3] = VT4;
    6391          60 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
    6392          20 :     VTListMap.InsertNode(Result, IP);
    6393             :   }
    6394         192 :   return Result->getSDVTList();
    6395             : }
    6396             : 
    6397     1063683 : SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
    6398     1063683 :   unsigned NumVTs = VTs.size();
    6399     2127366 :   FoldingSetNodeID ID;
    6400     1063683 :   ID.AddInteger(NumVTs);
    6401     3067765 :   for (unsigned index = 0; index < NumVTs; index++) {
    6402     6012246 :     ID.AddInteger(VTs[index].getRawBits());
    6403             :   }
    6404             : 
    6405     1063683 :   void *IP = nullptr;
    6406     2127366 :   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
    6407     1063683 :   if (!Result) {
    6408       62674 :     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
    6409       62674 :     std::copy(VTs.begin(), VTs.end(), Array);
    6410       94011 :     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
    6411       31337 :     VTListMap.InsertNode(Result, IP);
    6412             :   }
    6413     3191049 :   return Result->getSDVTList();
    6414             : }
    6415             : 
    6416             : 
    6417             : /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
    6418             : /// specified operands.  If the resultant node already exists in the DAG,
    6419             : /// this does not modify the specified node, instead it returns the node that
    6420             : /// already exists.  If the resultant node does not exist in the DAG, the
    6421             : /// input node is returned.  As a degenerate case, if you specify the same
    6422             : /// input operands as the node already has, the input node is returned.
    6423        1666 : SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
    6424             :   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
    6425             : 
    6426             :   // Check to see if there is no change.
    6427        4998 :   if (Op == N->getOperand(0)) return N;
    6428             : 
    6429             :   // See if the modified node already exists.
    6430        1666 :   void *InsertPos = nullptr;
    6431        1666 :   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
    6432             :     return Existing;
    6433             : 
    6434             :   // Nope it doesn't.  Remove the node from its current place in the maps.
    6435        1666 :   if (InsertPos)
    6436        1666 :     if (!RemoveNodeFromCSEMaps(N))
    6437           0 :       InsertPos = nullptr;
    6438             : 
    6439             :   // Now we update the operands.
    6440        1666 :   N->OperandList[0].set(Op);
    6441             : 
    6442             :   // If this gets put into a CSE map, add it.
    6443        1666 :   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
    6444             :   return N;
    6445             : }
    6446             : 
    6447       53199 : SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
    6448             :   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
    6449             : 
    6450             :   // Check to see if there is no change.
    6451      186008 :   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
    6452             :     return N;   // No operands changed, just return the input node.
    6453             : 
    6454             :   // See if the modified node already exists.
    6455       53131 :   void *InsertPos = nullptr;
    6456       53131 :   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
    6457             :     return Existing;
    6458             : 
    6459             :   // Nope it doesn't.  Remove the node from its current place in the maps.
    6460       52317 :   if (InsertPos)
    6461       52317 :     if (!RemoveNodeFromCSEMaps(N))
    6462           0 :       InsertPos = nullptr;
    6463             : 
    6464             :   // Now we update the operands.
    6465       65003 :   if (N->OperandList[0] != Op1)
    6466       12686 :     N->OperandList[0].set(Op1);
    6467       97854 :   if (N->OperandList[1] != Op2)
    6468       45537 :     N->OperandList[1].set(Op2);
    6469             : 
    6470             :   // If this gets put into a CSE map, add it.
    6471       52317 :   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
    6472             :   return N;
    6473             : }
    6474             : 
    6475       43441 : SDNode *SelectionDAG::
    6476             : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
    6477       43441 :   SDValue Ops[] = { Op1, Op2, Op3 };
    6478       43441 :   return UpdateNodeOperands(N, Ops);
    6479             : }
    6480             : 
    6481         303 : SDNode *SelectionDAG::
    6482             : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
    6483             :                    SDValue Op3, SDValue Op4) {
    6484         303 :   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
    6485         303 :   return UpdateNodeOperands(N, Ops);
    6486             : }
    6487             : 
    6488        1962 : SDNode *SelectionDAG::
    6489             : UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
    6490             :                    SDValue Op3, SDValue Op4, SDValue Op5) {
    6491        1962 :   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
    6492        1962 :   return UpdateNodeOperands(N, Ops);
    6493             : }
    6494             : 
    6495     3501057 : SDNode *SelectionDAG::
    6496             : UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
    6497     3501057 :   unsigned NumOps = Ops.size();
    6498             :   assert(N->getNumOperands() == NumOps &&
    6499             :          "Update with wrong number of operands");
    6500             : 
    6501             :   // If no operands changed just return the input node.
    6502    10503171 :   if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
    6503             :     return N;
    6504             : 
    6505             :   // See if the modified node already exists.
    6506      101831 :   void *InsertPos = nullptr;
    6507      101831 :   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
    6508             :     return Existing;
    6509             : 
    6510             :   // Nope it doesn't.  Remove the node from its current place in the maps.
    6511      101302 :   if (InsertPos)
    6512       91092 :     if (!RemoveNodeFromCSEMaps(N))
    6513           0 :       InsertPos = nullptr;
    6514             : 
    6515             :   // Now we update the operands.
    6516      747412 :   for (unsigned i = 0; i != NumOps; ++i)
    6517      816426 :     if (N->OperandList[i] != Ops[i])
    6518      170316 :       N->OperandList[i].set(Ops[i]);
    6519             : 
    6520             :   // If this gets put into a CSE map, add it.
    6521      101302 :   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
    6522             :   return N;
    6523             : }
    6524             : 
    6525             : /// DropOperands - Release the operands and set this node to have
    6526             : /// zero operands.
    6527     9286108 : void SDNode::DropOperands() {
    6528             :   // Unlike the code in MorphNodeTo that does this, we don't need to
    6529             :   // watch for dead nodes here.
    6530    33042107 :   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
    6531    14469891 :     SDUse &Use = *I++;
    6532    14469891 :     Use.set(SDValue());
    6533             :   }
    6534     9286108 : }
    6535             : 
    6536             : /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
    6537             : /// machine opcode.
    6538             : ///
    6539        5673 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6540             :                                    EVT VT) {
    6541        5673 :   SDVTList VTs = getVTList(VT);
    6542        5673 :   return SelectNodeTo(N, MachineOpc, VTs, None);
    6543             : }
    6544             : 
    6545         177 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6546             :                                    EVT VT, SDValue Op1) {
    6547         177 :   SDVTList VTs = getVTList(VT);
    6548         177 :   SDValue Ops[] = { Op1 };
    6549         177 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6550             : }
    6551             : 
    6552        1709 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6553             :                                    EVT VT, SDValue Op1,
    6554             :                                    SDValue Op2) {
    6555        1709 :   SDVTList VTs = getVTList(VT);
    6556        1709 :   SDValue Ops[] = { Op1, Op2 };
    6557        1709 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6558             : }
    6559             : 
    6560          66 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6561             :                                    EVT VT, SDValue Op1,
    6562             :                                    SDValue Op2, SDValue Op3) {
    6563          66 :   SDVTList VTs = getVTList(VT);
    6564          66 :   SDValue Ops[] = { Op1, Op2, Op3 };
    6565          66 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6566             : }
    6567             : 
    6568        3803 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6569             :                                    EVT VT, ArrayRef<SDValue> Ops) {
    6570        3803 :   SDVTList VTs = getVTList(VT);
    6571        3803 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6572             : }
    6573             : 
    6574           2 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6575             :                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
    6576           2 :   SDVTList VTs = getVTList(VT1, VT2);
    6577           2 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6578             : }
    6579             : 
    6580           4 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6581             :                                    EVT VT1, EVT VT2) {
    6582           4 :   SDVTList VTs = getVTList(VT1, VT2);
    6583           4 :   return SelectNodeTo(N, MachineOpc, VTs, None);
    6584             : }
    6585             : 
    6586           5 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6587             :                                    EVT VT1, EVT VT2, EVT VT3,
    6588             :                                    ArrayRef<SDValue> Ops) {
    6589           5 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    6590           5 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6591             : }
    6592             : 
    6593           0 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6594             :                                    EVT VT1, EVT VT2,
    6595             :                                    SDValue Op1, SDValue Op2) {
    6596           0 :   SDVTList VTs = getVTList(VT1, VT2);
    6597           0 :   SDValue Ops[] = { Op1, Op2 };
    6598           0 :   return SelectNodeTo(N, MachineOpc, VTs, Ops);
    6599             : }
    6600             : 
    6601       29886 : SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
    6602             :                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
    6603       29886 :   SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
    6604             :   // Reset the NodeID to -1.
    6605       59772 :   New->setNodeId(-1);
    6606       29886 :   if (New != N) {
    6607          84 :     ReplaceAllUsesWith(N, New);
    6608          84 :     RemoveDeadNode(N);
    6609             :   }
    6610       29886 :   return New;
    6611             : }
    6612             : 
    6613             : /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
    6614             : /// the line number information on the merged node since it is not possible to
    6615             : /// preserve the information that operation is associated with multiple lines.
    6616             : /// This will make the debugger working better at -O0, were there is a higher
    6617             : /// probability having other instructions associated with that line.
    6618             : ///
    6619             : /// For IROrder, we keep the smaller of the two
    6620       97101 : SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
    6621      291303 :   DebugLoc NLoc = N->getDebugLoc();
    6622       97115 :   if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
    6623           0 :     N->setDebugLoc(DebugLoc());
    6624             :   }
    6625      291303 :   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
    6626      194202 :   N->setIROrder(Order);
    6627      194202 :   return N;
    6628             : }
    6629             : 
    6630             : /// MorphNodeTo - This *mutates* the specified node to have the specified
    6631             : /// return type, opcode, and operands.
    6632             : ///
    6633             : /// Note that MorphNodeTo returns the resultant node.  If there is already a
    6634             : /// node of the specified opcode and operands, it returns that node instead of
    6635             : /// the current one.  Note that the SDLoc need not be the same.
    6636             : ///
    6637             : /// Using MorphNodeTo is faster than creating a new node and swapping it in
    6638             : /// with ReplaceAllUsesWith both because it often avoids allocating a new
    6639             : /// node, and because it doesn't require CSE recalculation for any of
    6640             : /// the node's users.
    6641             : ///
    6642             : /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
    6643             : /// As a consequence it isn't appropriate to use from within the DAG combiner or
    6644             : /// the legalizer which maintain worklists that would need to be updated when
    6645             : /// deleting things.
    6646     2730627 : SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
    6647             :                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
    6648             :   // If an identical node already exists, use it.
    6649     2730627 :   void *IP = nullptr;
    6650     2730627 :   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
    6651     4265882 :     FoldingSetNodeID ID;
    6652     2140777 :     AddNodeIDNode(ID, Opc, VTs, Ops);
    6653     4281554 :     if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
    6654       31344 :       return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
    6655             :   }
    6656             : 
    6657     2714955 :   if (!RemoveNodeFromCSEMaps(N))
    6658      573331 :     IP = nullptr;
    6659             : 
    6660             :   // Start the morphing.
    6661     2714955 :   N->NodeType = Opc;
    6662     2714955 :   N->ValueList = VTs.VTs;
    6663     2714955 :   N->NumValues = VTs.NumVTs;
    6664             : 
    6665             :   // Clear the operands list, updating used nodes to remove this from their
    6666             :   // use list.  Keep track of any operands that become dead as a result.
    6667     2714955 :   SmallPtrSet<SDNode*, 16> DeadNodeSet;
    6668     5429910 :   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
    6669     8027907 :     SDUse &Use = *I++;
    6670     8027907 :     SDNode *Used = Use.getNode();
    6671     8027907 :     Use.set(SDValue());
    6672     8027907 :     if (Used->use_empty())
    6673     3307228 :       DeadNodeSet.insert(Used);
    6674             :   }
    6675             : 
    6676             :   // For MachineNode, initialize the memory references information.
    6677     2707839 :   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
    6678     2707839 :     MN->setMemRefs(nullptr, nullptr);
    6679             : 
    6680             :   // Swap for an appropriately sized array from the recycler.
    6681     2714955 :   removeOperands(N);
    6682     2714955 :   createOperands(N, Ops);
    6683             : 
    6684             :   // Delete any nodes that are still dead after adding the uses for the
    6685             :   // new operands.
    6686     2714955 :   if (!DeadNodeSet.empty()) {
    6687     4565398 :     SmallVector<SDNode *, 16> DeadNodes;
    6688     5589927 :     for (SDNode *N : DeadNodeSet)
    6689     3307228 :       if (N->use_empty())
    6690     1217488 :         DeadNodes.push_back(N);
    6691     2282699 :     RemoveDeadNodes(DeadNodes);
    6692             :   }
    6693             : 
    6694     2714955 :   if (IP)
    6695     2119511 :     CSEMap.InsertNode(N, IP);   // Memoize the new node.
    6696     2714955 :   return N;
    6697             : }
    6698             : 
    6699          40 : SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
    6700          80 :   unsigned OrigOpc = Node->getOpcode();
    6701             :   unsigned NewOpc;
    6702          40 :   bool IsUnary = false;
    6703          40 :   bool IsTernary = false;
    6704          40 :   switch (OrigOpc) {
    6705           0 :   default:
    6706           0 :     llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
    6707             :   case ISD::STRICT_FADD: NewOpc = ISD::FADD; break;
    6708           6 :   case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break;
    6709           2 :   case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break;
    6710           2 :   case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break;
    6711           0 :   case ISD::STRICT_FREM: NewOpc = ISD::FREM; break;
    6712           4 :   case ISD::STRICT_FMA: NewOpc = ISD::FMA; IsTernary = true; break;
    6713           2 :   case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; IsUnary = true; break;
    6714           2 :   case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break;
    6715           2 :   case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break;
    6716           2 :   case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; IsUnary = true; break;
    6717           2 :   case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; IsUnary = true; break;
    6718           2 :   case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; IsUnary = true; break;
    6719           2 :   case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; IsUnary = true; break;
    6720           2 :   case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; IsUnary = true; break;
    6721           2 :   case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; IsUnary = true; break;
    6722           2 :   case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; IsUnary = true; break;
    6723           2 :   case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; IsUnary = true; break;
    6724           2 :   case ISD::STRICT_FNEARBYINT:
    6725           2 :     NewOpc = ISD::FNEARBYINT;
    6726           2 :     IsUnary = true;
    6727           2 :     break;
    6728             :   }
    6729             : 
    6730             :   // We're taking this node out of the chain, so we need to re-link things.
    6731          80 :   SDValue InputChain = Node->getOperand(0);
    6732          40 :   SDValue OutputChain = SDValue(Node, 1);
    6733          40 :   ReplaceAllUsesOfValueWith(OutputChain, InputChain);
    6734             : 
    6735         120 :   SDVTList VTs = getVTList(Node->getOperand(1).getValueType());
    6736          40 :   SDNode *Res = nullptr;
    6737          40 :   if (IsUnary)
    6738          60 :     Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) });
    6739          20 :   else if (IsTernary)
    6740          20 :     Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
    6741           8 :                                            Node->getOperand(2),
    6742           8 :                                            Node->getOperand(3)});
    6743             :   else
    6744          64 :     Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
    6745          32 :                                            Node->getOperand(2) });
    6746             : 
    6747             :   // MorphNodeTo can operate in two ways: if an existing node with the
    6748             :   // specified operands exists, it can just return it.  Otherwise, it
    6749             :   // updates the node in place to have the requested operands.
    6750          40 :   if (Res == Node) {
    6751             :     // If we updated the node in place, reset the node ID.  To the isel,
    6752             :     // this should be just like a newly allocated machine node.
    6753          40 :     Res->setNodeId(-1);
    6754             :   } else {
    6755           0 :     ReplaceAllUsesWith(Node, Res);
    6756           0 :     RemoveDeadNode(Node);
    6757             :   }
    6758             : 
    6759          40 :   return Res;
    6760             : }
    6761             : 
    6762             : /// getMachineNode - These are used for target selectors to create a new node
    6763             : /// with specified return type(s), MachineInstr opcode, and operands.
    6764             : ///
    6765             : /// Note that getMachineNode returns the resultant node.  If there is already a
    6766             : /// node of the specified opcode and operands, it returns that node instead of
    6767             : /// the current one.
    6768        1192 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6769             :                                             EVT VT) {
    6770        1192 :   SDVTList VTs = getVTList(VT);
    6771        1192 :   return getMachineNode(Opcode, dl, VTs, None);
    6772             : }
    6773             : 
    6774       50954 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6775             :                                             EVT VT, SDValue Op1) {
    6776       50954 :   SDVTList VTs = getVTList(VT);
    6777       50954 :   SDValue Ops[] = { Op1 };
    6778       50954 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6779             : }
    6780             : 
    6781       81205 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6782             :                                             EVT VT, SDValue Op1, SDValue Op2) {
    6783       81205 :   SDVTList VTs = getVTList(VT);
    6784       81205 :   SDValue Ops[] = { Op1, Op2 };
    6785       81205 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6786             : }
    6787             : 
    6788        2530 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6789             :                                             EVT VT, SDValue Op1, SDValue Op2,
    6790             :                                             SDValue Op3) {
    6791        2530 :   SDVTList VTs = getVTList(VT);
    6792        2530 :   SDValue Ops[] = { Op1, Op2, Op3 };
    6793        2530 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6794             : }
    6795             : 
    6796       40741 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6797             :                                             EVT VT, ArrayRef<SDValue> Ops) {
    6798       40741 :   SDVTList VTs = getVTList(VT);
    6799       40741 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6800             : }
    6801             : 
    6802        7722 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6803             :                                             EVT VT1, EVT VT2, SDValue Op1,
    6804             :                                             SDValue Op2) {
    6805        7722 :   SDVTList VTs = getVTList(VT1, VT2);
    6806        7722 :   SDValue Ops[] = { Op1, Op2 };
    6807        7722 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6808             : }
    6809             : 
    6810          16 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6811             :                                             EVT VT1, EVT VT2, SDValue Op1,
    6812             :                                             SDValue Op2, SDValue Op3) {
    6813          16 :   SDVTList VTs = getVTList(VT1, VT2);
    6814          16 :   SDValue Ops[] = { Op1, Op2, Op3 };
    6815          16 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6816             : }
    6817             : 
    6818       70583 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6819             :                                             EVT VT1, EVT VT2,
    6820             :                                             ArrayRef<SDValue> Ops) {
    6821       70583 :   SDVTList VTs = getVTList(VT1, VT2);
    6822       70583 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6823             : }
    6824             : 
    6825          16 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6826             :                                             EVT VT1, EVT VT2, EVT VT3,
    6827             :                                             SDValue Op1, SDValue Op2) {
    6828          16 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    6829          16 :   SDValue Ops[] = { Op1, Op2 };
    6830          16 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6831             : }
    6832             : 
    6833          53 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6834             :                                             EVT VT1, EVT VT2, EVT VT3,
    6835             :                                             SDValue Op1, SDValue Op2,
    6836             :                                             SDValue Op3) {
    6837          53 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    6838          53 :   SDValue Ops[] = { Op1, Op2, Op3 };
    6839          53 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6840             : }
    6841             : 
    6842         356 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6843             :                                             EVT VT1, EVT VT2, EVT VT3,
    6844             :                                             ArrayRef<SDValue> Ops) {
    6845         356 :   SDVTList VTs = getVTList(VT1, VT2, VT3);
    6846         356 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6847             : }
    6848             : 
    6849        7761 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
    6850             :                                             ArrayRef<EVT> ResultTys,
    6851             :                                             ArrayRef<SDValue> Ops) {
    6852        7761 :   SDVTList VTs = getVTList(ResultTys);
    6853        7761 :   return getMachineNode(Opcode, dl, VTs, Ops);
    6854             : }
    6855             : 
    6856      412469 : MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
    6857             :                                             SDVTList VTs,
    6858             :                                             ArrayRef<SDValue> Ops) {
    6859      824938 :   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
    6860             :   MachineSDNode *N;
    6861      412469 :   void *IP = nullptr;
    6862             : 
    6863             :   if (DoCSE) {
    6864      692923 :     FoldingSetNodeID ID;
    6865      387176 :     AddNodeIDNode(ID, ~Opcode, VTs, Ops);
    6866      387176 :     IP = nullptr;
    6867      387176 :     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
    6868      244287 :       return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
    6869             :     }
    6870             :   }
    6871             : 
    6872             :   // Allocate a new MachineSDNode.
    6873      662080 :   N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
    6874      331040 :   createOperands(N, Ops);
    6875             : 
    6876      331040 :   if (DoCSE)
    6877      305747 :     CSEMap.InsertNode(N, IP);
    6878             : 
    6879      331040 :   InsertNode(N);
    6880      331040 :   return N;
    6881             : }
    6882             : 
    6883             : /// getTargetExtractSubreg - A convenience function for creating
    6884             : /// TargetOpcode::EXTRACT_SUBREG nodes.
    6885       39593 : SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
    6886             :                                              SDValue Operand) {
    6887       79186 :   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
    6888             :   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
    6889       39593 :                                   VT, Operand, SRIdxVal);
    6890       39593 :   return SDValue(Subreg, 0);
    6891             : }
    6892             : 
    6893             : /// getTargetInsertSubreg - A convenience function for creating
    6894             : /// TargetOpcode::INSERT_SUBREG nodes.
    6895         497 : SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
    6896             :                                             SDValue Operand, SDValue Subreg) {
    6897         994 :   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
    6898             :   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
    6899         497 :                                   VT, Operand, Subreg, SRIdxVal);
    6900         497 :   return SDValue(Result, 0);
    6901             : }
    6902             : 
    6903             : /// getNodeIfExists - Get the specified node if it's already available, or
    6904             : /// else return NULL.
    6905      569075 : SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
    6906             :                                       ArrayRef<SDValue> Ops,
    6907             :                                       const SDNodeFlags Flags) {
    6908      569075 :   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
    6909     1138133 :     FoldingSetNodeID ID;
    6910      569075 :     AddNodeIDNode(ID, Opcode, VTList, Ops);
    6911      569075 :     void *IP = nullptr;
    6912     1707225 :     if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
    6913          17 :       E->intersectFlagsWith(Flags);
    6914          34 :       return E;
    6915             :     }
    6916             :   }
    6917             :   return nullptr;
    6918             : }
    6919             : 
    6920             : /// getDbgValue - Creates a SDDbgValue node.
    6921             : ///
    6922             : /// SDNode
    6923       12685 : SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
    6924             :                                       SDNode *N, unsigned R, bool IsIndirect,
    6925             :                                       const DebugLoc &DL, unsigned O) {
    6926             :   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
    6927             :          "Expected inlined-at fields to agree");
    6928       12685 :   return new (DbgInfo->getAlloc())
    6929       63425 :       SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O);
    6930             : }
    6931             : 
    6932             : /// Constant
    6933        5878 : SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
    6934             :                                               DIExpression *Expr,
    6935             :                                               const Value *C,
    6936             :                                               const DebugLoc &DL, unsigned O) {
    6937             :   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
    6938             :          "Expected inlined-at fields to agree");
    6939       29390 :   return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O);
    6940             : }
    6941             : 
    6942             : /// FrameIndex
    6943        3120 : SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
    6944             :                                                 DIExpression *Expr, unsigned FI,
    6945             :                                                 const DebugLoc &DL,
    6946             :                                                 unsigned O) {
    6947             :   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
    6948             :          "Expected inlined-at fields to agree");
    6949       15600 :   return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, DL, O);
    6950             : }
    6951             : 
    6952             : namespace {
    6953             : 
    6954             : /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
    6955             : /// pointed to by a use iterator is deleted, increment the use iterator
    6956             : /// so that it doesn't dangle.
    6957             : ///
    6958     9989980 : class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
    6959             :   SDNode::use_iterator &UI;
    6960             :   SDNode::use_iterator &UE;
    6961             : 
    6962       24836 :   void NodeDeleted(SDNode *N, SDNode *E) override {
    6963             :     // Increment the iterator as needed.
    6964       50693 :     while (UI != UE && N == *UI)
    6965           0 :       ++UI;
    6966       24836 :   }
    6967             : 
    6968             : public:
    6969             :   RAUWUpdateListener(SelectionDAG &d,
    6970             :                      SDNode::use_iterator &ui,
    6971             :                      SDNode::use_iterator &ue)
    6972     9989980 :     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
    6973             : };
    6974             : 
    6975             : } // end anonymous namespace
    6976             : 
    6977             : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
    6978             : /// This can cause recursive merging of nodes in the DAG.
    6979             : ///
    6980             : /// This version assumes From has a single result value.
    6981             : ///
    6982     1907689 : void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
    6983     1907689 :   SDNode *From = FromN.getNode();
    6984             :   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
    6985             :          "Cannot replace with this method!");
    6986             :   assert(From != To.getNode() && "Cannot replace uses of with self");
    6987             : 
    6988             :   // Preserve Debug Values
    6989     1907689 :   TransferDbgValues(FromN, To);
    6990             : 
    6991             :   // Iterate over all the existing uses of From. New uses will be added
    6992             :   // to the beginning of the use list, which we avoid visiting.
    6993             :   // This specifically avoids visiting uses of From that arise while the
    6994             :   // replacement is happening, because any such uses would be the result
    6995             :   // of CSE: If an existing node looks like From after one of its operands
    6996             :   // is replaced by To, we don't want to replace of all its users with To
    6997             :   // too. See PR3018 for more info.
    6998     1907689 :   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
    6999     1907689 :   RAUWUpdateListener Listener(*this, UI, UE);
    7000     6527607 :   while (UI != UE) {
    7001     2309959 :     SDNode *User = *UI;
    7002             : 
    7003             :     // This node is about to morph, remove its old self from the CSE maps.
    7004     2309959 :     RemoveNodeFromCSEMaps(User);
    7005             : 
    7006             :     // A user can appear in a use list multiple times, and when this
    7007             :     // happens the uses are usually next to each other in the list.
    7008             :     // To help reduce the number of CSE recomputations, process all
    7009             :     // the uses of this user that we can find this way.
    7010             :     do {
    7011     2319553 :       SDUse &Use = UI.getUse();
    7012     2319553 :       ++UI;
    7013     2319553 :       Use.set(To);
    7014     2777170 :     } while (UI != UE && *UI == User);
    7015             : 
    7016             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7017             :     // already exists there, recursively merge the results together.
    7018     2309959 :     AddModifiedNodeToCSEMaps(User);
    7019             :   }
    7020             : 
    7021             :   // If we just RAUW'd the root, take note.
    7022     1909125 :   if (FromN == getRoot())
    7023        1436 :     setRoot(To);
    7024     1907689 : }
    7025             : 
    7026             : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
    7027             : /// This can cause recursive merging of nodes in the DAG.
    7028             : ///
    7029             : /// This version assumes that for each value of From, there is a
    7030             : /// corresponding value in To in the same position with the same type.
    7031             : ///
    7032     1330114 : void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
    7033             : #ifndef NDEBUG
    7034             :   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
    7035             :     assert((!From->hasAnyUseOfValue(i) ||
    7036             :             From->getValueType(i) == To->getValueType(i)) &&
    7037             :            "Cannot use this version of ReplaceAllUsesWith!");
    7038             : #endif
    7039             : 
    7040             :   // Handle the trivial case.
    7041     1330114 :   if (From == To)
    7042           0 :     return;
    7043             : 
    7044             :   // Preserve Debug Info. Only do this if there's a use.
    7045     4075212 :   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
    7046     1414984 :     if (From->hasAnyUseOfValue(i)) {
    7047             :       assert((i < To->getNumValues()) && "Invalid To location");
    7048     2673184 :       TransferDbgValues(SDValue(From, i), SDValue(To, i));
    7049             :     }
    7050             : 
    7051             :   // Iterate over just the existing users of From. See the comments in
    7052             :   // the ReplaceAllUsesWith above.
    7053     2660228 :   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
    7054     1330114 :   RAUWUpdateListener Listener(*this, UI, UE);
    7055     5089266 :   while (UI != UE) {
    7056     1879576 :     SDNode *User = *UI;
    7057             : 
    7058             :     // This node is about to morph, remove its old self from the CSE maps.
    7059     1879576 :     RemoveNodeFromCSEMaps(User);
    7060             : 
    7061             :     // A user can appear in a use list multiple times, and when this
    7062             :     // happens the uses are usually next to each other in the list.
    7063             :     // To help reduce the number of CSE recomputations, process all
    7064             :     // the uses of this user that we can find this way.
    7065             :     do {
    7066     1885559 :       SDUse &Use = UI.getUse();
    7067     1885559 :       ++UI;
    7068     1885559 :       Use.setNode(To);
    7069     2442057 :     } while (UI != UE && *UI == User);
    7070             : 
    7071             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7072             :     // already exists there, recursively merge the results together.
    7073     1879576 :     AddModifiedNodeToCSEMaps(User);
    7074             :   }
    7075             : 
    7076             :   // If we just RAUW'd the root, take note.
    7077     1330114 :   if (From == getRoot().getNode())
    7078       10698 :     setRoot(SDValue(To, getRoot().getResNo()));
    7079             : }
    7080             : 
    7081             : /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
    7082             : /// This can cause recursive merging of nodes in the DAG.
    7083             : ///
    7084             : /// This version can replace From with any result values.  To must match the
    7085             : /// number and types of values returned by From.
    7086      923121 : void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
    7087     1846242 :   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
    7088      505835 :     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
    7089             : 
    7090             :   // Preserve Debug Info.
    7091     2089598 :   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
    7092      836156 :     TransferDbgValues(SDValue(From, i), *To);
    7093             : 
    7094             :   // Iterate over just the existing users of From. See the comments in
    7095             :   // the ReplaceAllUsesWith above.
    7096      834572 :   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
    7097      417286 :   RAUWUpdateListener Listener(*this, UI, UE);
    7098     1863868 :   while (UI != UE) {
    7099      723291 :     SDNode *User = *UI;
    7100             : 
    7101             :     // This node is about to morph, remove its old self from the CSE maps.
    7102      723291 :     RemoveNodeFromCSEMaps(User);
    7103             : 
    7104             :     // A user can appear in a use list multiple times, and when this
    7105             :     // happens the uses are usually next to each other in the list.
    7106             :     // To help reduce the number of CSE recomputations, process all
    7107             :     // the uses of this user that we can find this way.
    7108             :     do {
    7109      727019 :       SDUse &Use = UI.getUse();
    7110      727019 :       const SDValue &ToOp = To[Use.getResNo()];
    7111      727019 :       ++UI;
    7112      727019 :       Use.set(ToOp);
    7113     1038571 :     } while (UI != UE && *UI == User);
    7114             : 
    7115             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7116             :     // already exists there, recursively merge the results together.
    7117      723291 :     AddModifiedNodeToCSEMaps(User);
    7118             :   }
    7119             : 
    7120             :   // If we just RAUW'd the root, take note.
    7121      417286 :   if (From == getRoot().getNode())
    7122          13 :     setRoot(SDValue(To[getRoot().getResNo()]));
    7123             : }
    7124             : 
    7125             : /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
    7126             : /// uses of other values produced by From.getNode() alone.  The Deleted
    7127             : /// vector is handled the same way as for ReplaceAllUsesWith.
    7128     2075972 : void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
    7129             :   // Handle the really simple, really trivial case efficiently.
    7130     2811723 :   if (From == To) return;
    7131             : 
    7132             :   // Handle the simple, trivial, case efficiently.
    7133     2075652 :   if (From.getNode()->getNumValues() == 1) {
    7134      735751 :     ReplaceAllUsesWith(From, To);
    7135      735751 :     return;
    7136             :   }
    7137             : 
    7138             :   // Preserve Debug Info.
    7139     1339901 :   TransferDbgValues(From, To);
    7140             : 
    7141             :   // Iterate over just the existing users of From. See the comments in
    7142             :   // the ReplaceAllUsesWith above.
    7143     1339901 :   SDNode::use_iterator UI = From.getNode()->use_begin(),
    7144     4019703 :                        UE = From.getNode()->use_end();
    7145     1339901 :   RAUWUpdateListener Listener(*this, UI, UE);
    7146     3393507 :   while (UI != UE) {
    7147     2053606 :     SDNode *User = *UI;
    7148     2053606 :     bool UserRemovedFromCSEMaps = false;
    7149             : 
    7150             :     // A user can appear in a use list multiple times, and when this
    7151             :     // happens the uses are usually next to each other in the list.
    7152             :     // To help reduce the number of CSE recomputations, process all
    7153             :     // the uses of this user that we can find this way.
    7154             :     do {
    7155     2120270 :       SDUse &Use = UI.getUse();
    7156             : 
    7157             :       // Skip uses of different values from the same node.
    7158     3037289 :       if (Use.getResNo() != From.getResNo()) {
    7159      917019 :         ++UI;
    7160      917019 :         continue;
    7161             :       }
    7162             : 
    7163             :       // If this node hasn't been modified yet, it's still in the CSE maps,
    7164             :       // so remove its old self from the CSE maps.
    7165     1203251 :       if (!UserRemovedFromCSEMaps) {
    7166     1202217 :         RemoveNodeFromCSEMaps(User);
    7167     1202217 :         UserRemovedFromCSEMaps = true;
    7168             :       }
    7169             : 
    7170     1203251 :       ++UI;
    7171     1203251 :       Use.set(To);
    7172     2948737 :     } while (UI != UE && *UI == User);
    7173             : 
    7174             :     // We are iterating over all uses of the From node, so if a use
    7175             :     // doesn't use the specific value, no changes are made.
    7176     2053606 :     if (!UserRemovedFromCSEMaps)
    7177      851389 :       continue;
    7178             : 
    7179             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7180             :     // already exists there, recursively merge the results together.
    7181     1202217 :     AddModifiedNodeToCSEMaps(User);
    7182             :   }
    7183             : 
    7184             :   // If we just RAUW'd the root, take note.
    7185     1364456 :   if (From == getRoot())
    7186       24555 :     setRoot(To);
    7187             : }
    7188             : 
    7189             : namespace {
    7190             : 
    7191             :   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
    7192             :   /// to record information about a use.
    7193             :   struct UseMemo {
    7194             :     SDNode *User;
    7195             :     unsigned Index;
    7196             :     SDUse *Use;
    7197             :   };
    7198             : 
    7199             :   /// operator< - Sort Memos by User.
    7200             :   bool operator<(const UseMemo &L, const UseMemo &R) {
    7201         894 :     return (intptr_t)L.User < (intptr_t)R.User;
    7202             :   }
    7203             : 
    7204             : } // end anonymous namespace
    7205             : 
    7206             : /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
    7207             : /// uses of other values produced by From.getNode() alone.  The same value
    7208             : /// may appear in both the From and To list.  The Deleted vector is
    7209             : /// handled the same way as for ReplaceAllUsesWith.
    7210         597 : void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
    7211             :                                               const SDValue *To,
    7212             :                                               unsigned Num){
    7213             :   // Handle the simple, trivial case efficiently.
    7214         597 :   if (Num == 1)
    7215           0 :     return ReplaceAllUsesOfValueWith(*From, *To);
    7216             : 
    7217         597 :   TransferDbgValues(*From, *To);
    7218             : 
    7219             :   // Read up all the uses and make records of them. This helps
    7220             :   // processing new uses that are introduced during the
    7221             :   // replacement process.
    7222        1194 :   SmallVector<UseMemo, 4> Uses;
    7223        1912 :   for (unsigned i = 0; i != Num; ++i) {
    7224        1315 :     unsigned FromResNo = From[i].getResNo();
    7225        1315 :     SDNode *FromNode = From[i].getNode();
    7226             :     for (SDNode::use_iterator UI = FromNode->use_begin(),
    7227        3461 :          E = FromNode->use_end(); UI != E; ++UI) {
    7228        2146 :       SDUse &Use = UI.getUse();
    7229        2146 :       if (Use.getResNo() == FromResNo) {
    7230        1190 :         UseMemo Memo = { *UI, i, &Use };
    7231        1190 :         Uses.push_back(Memo);
    7232             :       }
    7233             :     }
    7234             :   }
    7235             : 
    7236             :   // Sort the uses, so that all the uses from a given User are together.
    7237        1791 :   std::sort(Uses.begin(), Uses.end());
    7238             : 
    7239        2316 :   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
    7240        1719 :        UseIndex != UseIndexEnd; ) {
    7241             :     // We know that this user uses some value of From.  If it is the right
    7242             :     // value, update it.
    7243        2244 :     SDNode *User = Uses[UseIndex].User;
    7244             : 
    7245             :     // This node is about to morph, remove its old self from the CSE maps.
    7246        1122 :     RemoveNodeFromCSEMaps(User);
    7247             : 
    7248             :     // The Uses array is sorted, so all the uses for a given User
    7249             :     // are next to each other in the list.
    7250             :     // To help reduce the number of CSE recomputations, process all
    7251             :     // the uses of this user that we can find this way.
    7252             :     do {
    7253        2380 :       unsigned i = Uses[UseIndex].Index;
    7254        2380 :       SDUse &Use = *Uses[UseIndex].Use;
    7255        1190 :       ++UseIndex;
    7256             : 
    7257        1190 :       Use.set(To[i]);
    7258        1783 :     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
    7259             : 
    7260             :     // Now that we have modified User, add it back to the CSE maps.  If it
    7261             :     // already exists there, recursively merge the results together.
    7262        1122 :     AddModifiedNodeToCSEMaps(User);
    7263             :   }
    7264             : }
    7265             : 
    7266             : /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
    7267             : /// based on their topological order. It returns the maximum id and a vector
    7268             : /// of the SDNodes* in assigned order by reference.
    7269      661399 : unsigned SelectionDAG::AssignTopologicalOrder() {
    7270      661399 :   unsigned DAGSize = 0;
    7271             : 
    7272             :   // SortedPos tracks the progress of the algorithm. Nodes before it are
    7273             :   // sorted, nodes after it are unsorted. When the algorithm completes
    7274             :   // it is at the end of the list.
    7275      661399 :   allnodes_iterator SortedPos = allnodes_begin();
    7276             : 
    7277             :   // Visit all the nodes. Move nodes with no operands to the front of
    7278             :   // the list immediately. Annotate nodes that do have operands with their
    7279             :   // operand count. Before we do this, the Node Id fields of the nodes
    7280             :   // may contain arbitrary values. After, the Node Id fields for nodes
    7281             :   // before SortedPos will contain the topological sort index, and the
    7282             :   // Node Id fields for nodes At SortedPos and after will contain the
    7283             :   // count of outstanding operands.
    7284     1322798 :   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
    7285    60524964 :     SDNode *N = &*I++;
    7286    20174988 :     checkForCycles(N, this);
    7287    20174988 :     unsigned Degree = N->getNumOperands();
    7288    20174988 :     if (Degree == 0) {
    7289             :       // A node with no uses, add it to the result array immediately.
    7290    16297358 :       N->setNodeId(DAGSize++);
    7291     8148679 :       allnodes_iterator Q(N);
    7292     8148679 :       if (Q != SortedPos)
    7293     9945489 :         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
    7294             :       assert(SortedPos != AllNodes.end() && "Overran node list");
    7295     8148679 :       ++SortedPos;
    7296             :     } else {
    7297             :       // Temporarily use the Node Id as scratch space for the degree count.
    7298    12026309 :       N->setNodeId(Degree);
    7299             :     }
    7300             :   }
    7301             : 
    7302             :   // Visit all the nodes. As we iterate, move nodes into sorted order,
    7303             :   // such that by the time the end is reached all nodes will be sorted.
    7304    20836387 :   for (SDNode &Node : allnodes()) {
    7305    20174988 :     SDNode *N = &Node;
    7306    20174988 :     checkForCycles(N, this);
    7307             :     // N is in sorted position, so all its uses have one less operand
    7308             :     // that needs to be sorted.
    7309    20174988 :     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
    7310    54395957 :          UI != UE; ++UI) {
    7311    34220969 :       SDNode *P = *UI;
    7312    34220969 :       unsigned Degree = P->getNodeId();
    7313             :       assert(Degree != 0 && "Invalid node degree");
    7314    34220969 :       --Degree;
    7315    34220969 :       if (Degree == 0) {
    7316             :         // All of P's operands are sorted, so P may sorted now.
    7317    24052618 :         P->setNodeId(DAGSize++);
    7318    24052618 :         if (P->getIterator() != SortedPos)
    7319    12242634 :           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
    7320             :         assert(SortedPos != AllNodes.end() && "Overran node list");
    7321             :         ++SortedPos;
    7322             :       } else {
    7323             :         // Update P's outstanding operand count.
    7324    22194660 :         P->setNodeId(Degree);
    7325             :       }
    7326             :     }
    7327    40349976 :     if (Node.getIterator() == SortedPos) {
    7328             : #ifndef NDEBUG
    7329             :       allnodes_iterator I(N);
    7330             :       SDNode *S = &*++I;
    7331             :       dbgs() << "Overran sorted position:\n";
    7332             :       S->dumprFull(this); dbgs() << "\n";
    7333             :       dbgs() << "Checking if this is due to cycles\n";
    7334             :       checkForCycles(this, true);
    7335             : #endif
    7336           0 :       llvm_unreachable(nullptr);
    7337             :     }
    7338             :   }
    7339             : 
    7340             :   assert(SortedPos == AllNodes.end() &&
    7341             :          "Topological sort incomplete!");
    7342             :   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
    7343             :          "First node in topological sort is not the entry token!");
    7344             :   assert(AllNodes.front().getNodeId() == 0 &&
    7345             :          "First node in topological sort has non-zero id!");
    7346             :   assert(AllNodes.front().getNumOperands() == 0 &&
    7347             :          "First node in topological sort has operands!");
    7348             :   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
    7349             :          "Last node in topologic sort has unexpected id!");
    7350             :   assert(AllNodes.back().use_empty() &&
    7351             :          "Last node in topologic sort has users!");
    7352             :   assert(DAGSize == allnodes_size() && "Node count mismatch!");
    7353      661399 :   return DAGSize;
    7354             : }
    7355             : 
    7356             : /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
    7357             : /// value is produced by SD.
    7358       21683 : void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
    7359       21683 :   if (SD) {
    7360             :     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
    7361             :     SD->setHasDebugValue(true);
    7362             :   }
    7363       21683 :   DbgInfo->add(DB, SD, isParameter);
    7364       21683 : }
    7365             : 
    7366             : /// TransferDbgValues - Transfer SDDbgValues. Called in replace nodes.
    7367     5420935 : void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
    7368    10841786 :   if (From == To || !From.getNode()->getHasDebugValue())
    7369     5419891 :     return;
    7370        1044 :   SDNode *FromNode = From.getNode();
    7371        1044 :   SDNode *ToNode = To.getNode();
    7372        2088 :   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
    7373        2088 :   SmallVector<SDDbgValue *, 2> ClonedDVs;
    7374        4191 :   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
    7375        3147 :        I != E; ++I) {
    7376        2103 :     SDDbgValue *Dbg = *I;
    7377             :     // Only add Dbgvalues attached to same ResNo.
    7378        4206 :     if (Dbg->getKind() == SDDbgValue::SDNODE &&
    7379        4206 :         Dbg->getSDNode() == From.getNode() &&
    7380        5829 :         Dbg->getResNo() == From.getResNo() && !Dbg->isInvalidated()) {
    7381             :       assert(FromNode != ToNode &&
    7382             :              "Should not transfer Debug Values intranode");
    7383        3246 :       SDDbgValue *Clone = getDbgValue(Dbg->getVariable(), Dbg->getExpression(),
    7384        1623 :                                       ToNode, To.getResNo(), Dbg->isIndirect(),
    7385        4869 :                                       Dbg->getDebugLoc(), Dbg->getOrder());
    7386        1623 :       ClonedDVs.push_back(Clone);
    7387        1623 :       Dbg->setIsInvalidated();
    7388             :     }
    7389             :   }
    7390        4755 :   for (SDDbgValue *I : ClonedDVs)
    7391        1623 :     AddDbgValue(I, ToNode, false);
    7392             : }
    7393             : 
    7394        1354 : SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
    7395             :                                                    SDValue NewMemOp) {
    7396             :   assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
    7397             :   // The new memory operation must have the same position as the old load in
    7398             :   // terms of memory dependency. Create a TokenFactor for the old load and new
    7399             :   // memory operation and update uses of the old load's output chain to use that
    7400             :   // TokenFactor.
    7401        1354 :   SDValue OldChain = SDValue(OldLoad, 1);
    7402        2708 :   SDValue NewChain = SDValue(NewMemOp.getNode(), 1);
    7403        1354 :   if (!OldLoad->hasAnyUseOfValue(1))
    7404        1141 :     return NewChain;
    7405             : 
    7406             :   SDValue TokenFactor =
    7407         852 :       getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain);
    7408         213 :   ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
    7409         213 :   UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain);
    7410         213 :   return TokenFactor;
    7411             : }
    7412             : 
    7413             : //===----------------------------------------------------------------------===//
    7414             : //                              SDNode Class
    7415             : //===----------------------------------------------------------------------===//
    7416             : 
    7417    16547987 : bool llvm::isNullConstant(SDValue V) {
    7418     2423315 :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
    7419     2423315 :   return Const != nullptr && Const->isNullValue();
    7420             : }
    7421             : 
    7422    13545874 : bool llvm::isNullFPConstant(SDValue V) {
    7423       25684 :   ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
    7424       45058 :   return Const != nullptr && Const->isZero() && !Const->isNegative();
    7425             : }
    7426             : 
    7427      584320 : bool llvm::isAllOnesConstant(SDValue V) {
    7428      450456 :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
    7429      450456 :   return Const != nullptr && Const->isAllOnesValue();
    7430             : }
    7431             : 
    7432      147539 : bool llvm::isOneConstant(SDValue V) {
    7433       90044 :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
    7434       90044 :   return Const != nullptr && Const->isOne();
    7435             : }
    7436             : 
    7437      128381 : bool llvm::isBitwiseNot(SDValue V) {
    7438      257248 :   return V.getOpcode() == ISD::XOR && isAllOnesConstant(V.getOperand(1));
    7439             : }
    7440             : 
    7441     3452796 : ConstantSDNode *llvm::isConstOrConstSplat(SDValue N) {
    7442             :   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
    7443             :     return CN;
    7444             : 
    7445       28222 :   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
    7446       34652 :     BitVector UndefElements;
    7447       28222 :     ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
    7448             : 
    7449             :     // BuildVectors can truncate their operands. Ignore that case here.
    7450             :     // FIXME: We blindly ignore splats which include undef which is overly
    7451             :     // pessimistic.
    7452       51710 :     if (CN && UndefElements.none() &&
    7453       98029 :         CN->getValueType(0) == N.getValueType().getScalarType())
    7454       43584 :       return CN;
    7455             :   }
    7456             : 
    7457             :   return nullptr;
    7458             : }
    7459             : 
    7460      157561 : ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N) {
    7461             :   if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
    7462             :     return CN;
    7463             : 
    7464        4172 :   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
    7465        7046 :     BitVector UndefElements;
    7466        4172 :     ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
    7467             : 
    7468        5479 :     if (CN && UndefElements.none())
    7469        2596 :       return CN;
    7470             :   }
    7471             : 
    7472             :   return nullptr;
    7473             : }
    7474             : 
    7475    11173785 : HandleSDNode::~HandleSDNode() {
    7476     3724595 :   DropOperands();
    7477     3724595 : }
    7478             : 
    7479     1389587 : GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
    7480             :                                          const DebugLoc &DL,
    7481             :                                          const GlobalValue *GA, EVT VT,
    7482     1389587 :                                          int64_t o, unsigned char TF)
    7483     5558348 :     : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
    7484     1389587 :   TheGlobal = GA;
    7485     1389587 : }
    7486             : 
    7487         199 : AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
    7488             :                                          EVT VT, unsigned SrcAS,
    7489         199 :                                          unsigned DestAS)
    7490             :     : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
    7491         796 :       SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
    7492             : 
    7493     3633986 : MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
    7494     3633986 :                      SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
    7495    10901958 :     : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
    7496     7267972 :   MemSDNodeBits.IsVolatile = MMO->isVolatile();
    7497     7267972 :   MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
    7498     7267972 :   MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
    7499     7267972 :   MemSDNodeBits.IsInvariant = MMO->isInvariant();
    7500             : 
    7501             :   // We check here that the size of the memory operand fits within the size of
    7502             :   // the MMO. This is because the MMO might indicate only a possible address
    7503             :   // range instead of specifying the affected memory addresses precisely.
    7504             :   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
    7505     3633986 : }
    7506             : 
    7507             : /// Profile - Gather unique data for the node.
    7508             : ///
    7509    33131107 : void SDNode::Profile(FoldingSetNodeID &ID) const {
    7510    33131107 :   AddNodeIDNode(ID, this);
    7511    33131107 : }
    7512             : 
    7513             : namespace {
    7514             : 
    7515       32740 :   struct EVTArray {
    7516             :     std::vector<EVT> VTs;
    7517             : 
    7518       33296 :     EVTArray() {
    7519       16648 :       VTs.reserve(MVT::LAST_VALUETYPE);
    7520     1881224 :       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
    7521     5593728 :         VTs.push_back(MVT((MVT::SimpleValueType)i));
    7522       16648 :     }
    7523             :   };
    7524             : 
    7525             : } // end anonymous namespace
    7526             : 
    7527             : static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
    7528             : static ManagedStatic<EVTArray> SimpleVTArray;
    7529             : static ManagedStatic<sys::SmartMutex<true>> VTMutex;
    7530             : 
    7531             : /// getValueTypeList - Return a pointer to the specified value type.
    7532             : ///
    7533    39197730 : const EVT *SDNode::getValueTypeList(EVT VT) {
    7534    39197730 :   if (VT.isExtended()) {
    7535      219369 :     sys::SmartScopedLock<true> Lock(*VTMutex);
    7536      219369 :     return &(*EVTs->insert(VT).first);
    7537             :   } else {
    7538             :     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
    7539             :            "Value type out of range!");
    7540   117373821 :     return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
    7541             :   }
    7542             : }
    7543             : 
    7544             : /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
    7545             : /// indicated value.  This method ignores uses of other values defined by this
    7546             : /// operation.
    7547    14421481 : bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
    7548             :   assert(Value < getNumValues() && "Bad value!");
    7549             : 
    7550             :   // TODO: Only iterate over uses of a given value of the node
    7551    44838917 :   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
    7552    36754438 :     if (UI.getUse().getResNo() == Value) {
    7553    16802215 :       if (NUses == 0)
    7554             :         return false;
    7555    14420951 :       --NUses;
    7556             :     }
    7557             :   }
    7558             : 
    7559             :   // Found exactly the right number of uses?
    7560    12040217 :   return NUses == 0;
    7561             : }
    7562             : 
    7563             : /// hasAnyUseOfValue - Return true if there are any use of the indicated
    7564             : /// value. This method ignores uses of other values defined by this operation.
    7565     7974755 : bool SDNode::hasAnyUseOfValue(unsigned Value) const {
    7566             :   assert(Value < getNumValues() && "Bad value!");
    7567             : 
    7568    19288905 :   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
    7569    18891704 :     if (UI.getUse().getResNo() == Value)
    7570             :       return true;
    7571             : 
    7572             :   return false;
    7573             : }
    7574             : 
    7575             : /// isOnlyUserOf - Return true if this node is the only use of N.
    7576       91266 : bool SDNode::isOnlyUserOf(const SDNode *N) const {
    7577       91266 :   bool Seen = false;
    7578      283033 :   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
    7579      112708 :     SDNode *User = *I;
    7580      112708 :     if (User == this)
    7581      100501 :       Seen = true;
    7582             :     else
    7583             :       return false;
    7584             :   }
    7585             : 
    7586             :   return Seen;
    7587             : }
    7588             : 
    7589             : /// Return true if the only users of N are contained in Nodes.
    7590       46316 : bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
    7591       46316 :   bool Seen = false;
    7592      119083 :   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
    7593       69971 :     SDNode *User = *I;
    7594      139942 :     if (llvm::any_of(Nodes,
    7595      144612 :                      [&User](const SDNode *Node) { return User == Node; }))
    7596       26451 :       Seen = true;
    7597             :     else
    7598       43520 :       return false;
    7599             :   }
    7600             : 
    7601             :   return Seen;
    7602             : }
    7603             : 
    7604             : /// isOperand - Return true if this node is an operand of N.
    7605      607495 : bool SDValue::isOperandOf(const SDNode *N) const {
    7606     6577251 :   for (const SDValue &Op : N->op_values())
    7607     2675686 :     if (*this == Op)
    7608             :       return true;
    7609             :   return false;
    7610             : }
    7611             : 
    7612        2284 : bool SDNode::isOperandOf(const SDNode *N) const {
    7613       14091 :   for (const SDValue &Op : N->op_values())
    7614        5623 :     if (this == Op.getNode())
    7615             :       return true;
    7616             :   return false;
    7617             : }
    7618             : 
    7619             : /// reachesChainWithoutSideEffects - Return true if this operand (which must
    7620             : /// be a chain) reaches the specified operand without crossing any
    7621             : /// side-effecting instructions on any chain path.  In practice, this looks
    7622             : /// through token factors and non-volatile loads.  In order to remain efficient,
    7623             : /// this only looks a couple of nodes in, it does not do an exhaustive search.
    7624             : ///
    7625             : /// Note that we only need to examine chains when we're searching for
    7626             : /// side-effects; SelectionDAG requires that all side-effects are represented
    7627             : /// by chains, even if another operand would force a specific ordering. This
    7628             : /// constraint is necessary to allow transformations like splitting loads.
    7629        2550 : bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
    7630             :                                              unsigned Depth) const {
    7631         818 :   if (*this == Dest) return true;
    7632             : 
    7633             :   // Don't search too deeply, we just want to be able to see through
    7634             :   // TokenFactor's etc.
    7635         818 :   if (Depth == 0) return false;
    7636             : 
    7637             :   // If this is a token factor, all inputs to the TF happen in parallel.
    7638        1566 :   if (getOpcode() == ISD::TokenFactor) {
    7639             :     // First, try a shallow search.
    7640         162 :     if (is_contained((*this)->ops(), Dest)) {
    7641             :       // We found the chain we want as an operand of this TokenFactor.
    7642             :       // Essentially, we reach the chain without side-effects if we could
    7643             :       // serialize the TokenFactor into a simple chain of operations with
    7644             :       // Dest as the last operation. This is automatically true if the
    7645             :       // chain has one use: there are no other ordering constraints.
    7646             :       // If the chain has more than one use, we give up: some other
    7647             :       // use of Dest might force a side-effect between Dest and the current
    7648             :       // node.
    7649          66 :       if (Dest.hasOneUse())
    7650             :         return true;
    7651             :     }
    7652             :     // Next, try a deep search: check whether every operand of the TokenFactor
    7653             :     // reaches Dest.
    7654         117 :     return llvm::all_of((*this)->ops(), [=](SDValue Op) {
    7655          49 :       return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
    7656          90 :     });
    7657             :   }
    7658             : 
    7659             :   // Loads don't have side effects, look through them.
    7660          80 :   if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
    7661         160 :     if (!Ld->isVolatile())
    7662         160 :       return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
    7663             :   }
    7664             :   return false;
    7665             : }
    7666             : 
    7667        4921 : bool SDNode::hasPredecessor(const SDNode *N) const {
    7668        9842 :   SmallPtrSet<const SDNode *, 32> Visited;
    7669        9842 :   SmallVector<const SDNode *, 16> Worklist;
    7670        4921 :   Worklist.push_back(this);
    7671        9842 :   return hasPredecessorHelper(N, Visited, Worklist);
    7672             : }
    7673             : 
    7674      595044 : void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
    7675      595044 :   this->Flags.intersectWith(Flags);
    7676      595044 : }
    7677             : 
    7678        3186 : SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
    7679             :   assert(N->getNumValues() == 1 &&
    7680             :          "Can't unroll a vector with multiple results!");
    7681             : 
    7682        6372 :   EVT VT = N->getValueType(0);
    7683        3186 :   unsigned NE = VT.getVectorNumElements();
    7684        3186 :   EVT EltVT = VT.getVectorElementType();
    7685        6372 :   SDLoc dl(N);
    7686             : 
    7687        6372 :   SmallVector<SDValue, 8> Scalars;
    7688        9558 :   SmallVector<SDValue, 4> Operands(N->getNumOperands());
    7689             : 
    7690             :   // If ResNE is 0, fully unroll the vector op.
    7691        3186 :   if (ResNE == 0)
    7692             :     ResNE = NE;
    7693          15 :   else if (NE > ResNE)
    7694           0 :     NE = ResNE;
    7695             : 
    7696             :   unsigned i;
    7697       12968 :   for (i= 0; i != NE; ++i) {
    7698       35364 :     for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
    7699       31600 :       SDValue Operand = N->getOperand(j);
    7700       31600 :       EVT OperandVT = Operand.getValueType();
    7701       15800 :       if (OperandVT.isVector()) {
    7702             :         // A vector operand; extract a single element.
    7703       15394 :         EVT OperandEltVT = OperandVT.getVectorElementType();
    7704       30788 :         Operands[j] =
    7705       30788 :             getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
    7706       61576 :                     getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
    7707             :       } else {
    7708             :         // A scalar operand; just use it as is.
    7709         812 :         Operands[j] = Operand;
    7710             :       }
    7711             :     }
    7712             : 
    7713       19564 :     switch (N->getOpcode()) {
    7714        9264 :     default: {
    7715        9264 :       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
    7716       37056 :                                 N->getFlags()));
    7717        9264 :       break;
    7718             :     }
    7719          46 :     case ISD::VSELECT:
    7720          92 :       Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
    7721          46 :       break;
    7722         400 :     case ISD::SHL:
    7723             :     case ISD::SRA:
    7724             :     case ISD::SRL:
    7725             :     case ISD::ROTL:
    7726             :     case ISD::ROTR:
    7727         800 :       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
    7728         400 :                                getShiftAmountOperand(Operands[0].getValueType(),
    7729        2000 :                                                      Operands[1])));
    7730         400 :       break;
    7731          72 :     case ISD::SIGN_EXTEND_INREG:
    7732             :     case ISD::FP_ROUND_INREG: {
    7733         144 :       EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
    7734          72 :       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
    7735          72 :                                 Operands[0],
    7736         216 :                                 getValueType(ExtVT)));
    7737             :     }
    7738             :     }
    7739             :   }
    7740             : 
    7741        3200 :   for (; i < ResNE; ++i)
    7742           7 :     Scalars.push_back(getUNDEF(EltVT));
    7743             : 
    7744        3186 :   EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
    7745        6372 :   return getBuildVector(VecVT, dl, Scalars);
    7746             : }
    7747             : 
    7748        4752 : bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
    7749             :                                                   LoadSDNode *Base,
    7750             :                                                   unsigned Bytes,
    7751             :                                                   int Dist) const {
    7752       14210 :   if (LD->isVolatile() || Base->isVolatile())
    7753             :     return false;
    7754       13983 :   if (LD->isIndexed() || Base->isIndexed())
    7755             :     return false;
    7756       18311 :   if (LD->getChain() != Base->getChain())
    7757             :     return false;
    7758        8656 :   EVT VT = LD->getValueType(0);
    7759        4328 :   if (VT.getSizeInBits() / 8 != Bytes)
    7760             :     return false;
    7761             : 
    7762        8656 :   SDValue Loc = LD->getOperand(1);
    7763        8656 :   SDValue BaseLoc = Base->getOperand(1);
    7764             : 
    7765        4328 :   auto BaseLocDecomp = BaseIndexOffset::match(BaseLoc, *this);
    7766        4328 :   auto LocDecomp = BaseIndexOffset::match(Loc, *this);
    7767             : 
    7768        4328 :   int64_t Offset = 0;
    7769        4328 :   if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
    7770        4137 :     return (Dist * Bytes == Offset);
    7771             :   return false;
    7772             : }
    7773             : 
    7774             : /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
    7775             : /// it cannot be inferred.
    7776     5270335 : unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
    7777             :   // If this is a GlobalAddress + cst, return the alignment.
    7778             :   const GlobalValue *GV;
    7779     5270335 :   int64_t GVOffset = 0;
    7780     5270335 :   if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
    7781     4623735 :     unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
    7782     1544719 :     KnownBits Known(PtrWidth);
    7783     3082490 :     llvm::computeKnownBits(GV, Known, getDataLayout());
    7784     1541245 :     unsigned AlignBits = Known.countMinTrailingZeros();
    7785     3079016 :     unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
    7786     1537771 :     if (Align)
    7787     3075542 :       return MinAlign(Align, GVOffset);
    7788             :   }
    7789             : 
    7790             :   // If this is a direct reference to a stack slot, use information about the
    7791             :   // stack slot's alignment.
    7792     3732564 :   int FrameIdx = 1 << 31;
    7793     3732564 :   int64_t FrameOffset = 0;
    7794     1182999 :   if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
    7795     1182999 :     FrameIdx = FI->getIndex();
    7796     2549565 :   } else if (isBaseWithConstantOffset(Ptr) &&
    7797     3421608 :              isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
    7798             :     // Handle FI+Cst
    7799      819492 :     FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
    7800      546328 :     FrameOffset = Ptr.getConstantOperandVal(1);
    7801             :   }
    7802             : 
    7803     1456163 :   if (FrameIdx != (1 << 31)) {
    7804     1456163 :     const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
    7805     4368489 :     unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
    7806     1456163 :                                     FrameOffset);
    7807     1456163 :     return FIInfoAlign;
    7808             :   }
    7809             : 
    7810             :   return 0;
    7811             : }
    7812             : 
    7813             : /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
    7814             : /// which is split (or expanded) into two not necessarily identical pieces.
    7815       95374 : std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
    7816             :   // Currently all types are split in half.
    7817       95374 :   EVT LoVT, HiVT;
    7818       95374 :   if (!VT.isVector())
    7819        3234 :     LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
    7820             :   else
    7821       93757 :     LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
    7822             : 
    7823       95374 :   return std::make_pair(LoVT, HiVT);
    7824             : }
    7825             : 
    7826             : /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
    7827             : /// low/high part.
    7828             : std::pair<SDValue, SDValue>
    7829       15919 : SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
    7830             :                           const EVT &HiVT) {
    7831             :   assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
    7832             :          N.getValueType().getVectorNumElements() &&
    7833             :          "More vector elements requested than available!");
    7834       15919 :   SDValue Lo, Hi;
    7835       15919 :   Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
    7836       63676 :                getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
    7837       15919 :   Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
    7838       15919 :                getConstant(LoVT.getVectorNumElements(), DL,
    7839       79595 :                            TLI->getVectorIdxTy(getDataLayout())));
    7840       15919 :   return std::make_pair(Lo, Hi);
    7841             : }
    7842             : 
    7843       17476 : void SelectionDAG::ExtractVectorElements(SDValue Op,
    7844             :                                          SmallVectorImpl<SDValue> &Args,
    7845             :                                          unsigned Start, unsigned Count) {
    7846       34952 :   EVT VT = Op.getValueType();
    7847       17476 :   if (Count == 0)
    7848        4466 :     Count = VT.getVectorNumElements();
    7849             : 
    7850       17476 :   EVT EltVT = VT.getVectorElementType();
    7851       52428 :   EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
    7852       34952 :   SDLoc SL(Op);
    7853       92294 :   for (unsigned i = Start, e = Start + Count; i != e; ++i) {
    7854      149636 :     Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
    7855      149636 :                            Op, getConstant(i, SL, IdxTy)));
    7856             :   }
    7857       17476 : }
    7858             : 
    7859             : // getAddressSpace - Return the address space this GlobalAddress belongs to.
    7860        4863 : unsigned GlobalAddressSDNode::getAddressSpace() const {
    7861       14589 :   return getGlobal()->getType()->getAddressSpace();
    7862             : }
    7863             : 
    7864       20191 : Type *ConstantPoolSDNode::getType() const {
    7865       20191 :   if (isMachineConstantPoolEntry())
    7866         340 :     return Val.MachineCPVal->getType();
    7867       19851 :   return Val.ConstVal->getType();
    7868             : }
    7869             : 
    7870      290337 : bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
    7871             :                                         unsigned &SplatBitSize,
    7872             :                                         bool &HasAnyUndefs,
    7873             :                                         unsigned MinSplatBits,
    7874             :                                         bool IsBigEndian) const {
    7875      580674 :   EVT VT = getValueType(0);
    7876             :   assert(VT.isVector() && "Expected a vector type");
    7877      290337 :   unsigned VecWidth = VT.getSizeInBits();
    7878      290337 :   if (MinSplatBits > VecWidth)
    7879             :     return false;
    7880             : 
    7881             :   // FIXME: The widths are based on this node's type, but build vectors can
    7882             :   // truncate their operands.
    7883      871011 :   SplatValue = APInt(VecWidth, 0);
    7884      871011 :   SplatUndef = APInt(VecWidth, 0);
    7885             : 
    7886             :   // Get the bits. Bits with undefined values (when the corresponding element
    7887             :   // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
    7888             :   // in SplatValue. If any of the values are not constant, give up and return
    7889             :   // false.
    7890      580674 :   unsigned int NumOps = getNumOperands();
    7891             :   assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
    7892      290337 :   unsigned EltWidth = VT.getScalarSizeInBits();
    7893             : 
    7894     1056289 :   for (unsigned j = 0; j < NumOps; ++j) {
    7895      772827 :     unsigned i = IsBigEndian ? NumOps - 1 - j : j;
    7896     1545654 :     SDValue OpVal = getOperand(i);
    7897      772827 :     unsigned BitPos = j * EltWidth;
    7898             : 
    7899     1545654 :     if (OpVal.isUndef())
    7900       10583 :       SplatUndef.setBits(BitPos, BitPos + EltWidth);
    7901      751050 :     else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
    7902     1502100 :       SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
    7903        4319 :     else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
    7904        8638 :       SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
    7905             :     else
    7906             :       return false;
    7907             :   }
    7908             : 
    7909             :   // The build_vector is all constants or undefs. Find the smallest element
    7910             :   // size that splats the vector.
    7911      283462 :   HasAnyUndefs = (SplatUndef != 0);
    7912             : 
    7913             :   // FIXME: This does not work for vectors with elements less than 8 bits.
    7914      580845 :   while (VecWidth > 8) {
    7915      576799 :     unsigned HalfSize = VecWidth / 2;
    7916     1450981 :     APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
    7917      874182 :     APInt LowValue = SplatValue.trunc(HalfSize);
    7918     1450981 :     APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
    7919      874182 :     APInt LowUndef = SplatUndef.trunc(HalfSize);
    7920             : 
    7921             :     // If the two halves do not match (ignoring undef bits), stop here.
    7922     8075186 :     if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
    7923             :         MinSplatBits > HalfSize)
    7924             :       break;
    7925             : 
    7926     1486915 :     SplatValue = HighValue | LowValue;
    7927     1486915 :     SplatUndef = HighUndef & LowUndef;
    7928             : 
    7929      297383 :     VecWidth = HalfSize;
    7930             :   }
    7931             : 
    7932      283462 :   SplatBitSize = VecWidth;
    7933      283462 :   return true;
    7934             : }
    7935             : 
    7936       74825 : SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
    7937       74825 :   if (UndefElements) {
    7938       57838 :     UndefElements->clear();
    7939      115676 :     UndefElements->resize(getNumOperands());
    7940             :   }
    7941       74825 :   SDValue Splatted;
    7942      700231 :   for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
    7943     1143038 :     SDValue Op = getOperand(i);
    7944     1143038 :     if (Op.isUndef()) {
    7945        6352 :       if (UndefElements)
    7946       12234 :         (*UndefElements)[i] = true;
    7947      565167 :     } else if (!Splatted) {
    7948             :       Splatted = Op;
    7949       20938 :     } else if (Splatted != Op) {
    7950       20938 :       return SDValue();
    7951             :     }
    7952             :   }
    7953             : 
    7954       53887 :   if (!Splatted) {
    7955             :     assert(getOperand(0).isUndef() &&
    7956             :            "Can only have a splat without a constant for all undefs.");
    7957          98 :     return getOperand(0);
    7958             :   }
    7959             : 
    7960       53838 :   return Splatted;
    7961             : }
    7962             : 
    7963             : ConstantSDNode *
    7964       44036 : BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
    7965       88072 :   return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
    7966             : }
    7967             : 
    7968             : ConstantFPSDNode *
    7969        4563 : BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
    7970        9126 :   return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
    7971             : }
    7972             : 
    7973             : int32_t
    7974          55 : BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
    7975             :                                                    uint32_t BitWidth) const {
    7976             :   if (ConstantFPSDNode *CN =
    7977          56 :           dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
    7978             :     bool IsExact;
    7979         108 :     APSInt IntVal(BitWidth);
    7980          54 :     const APFloat &APF = CN->getValueAPF();
    7981          54 :     if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
    7982         102 :             APFloat::opOK ||
    7983          48 :         !IsExact)
    7984             :       return -1;
    7985             : 
    7986          48 :     return IntVal.exactLogBase2();
    7987             :   }
    7988             :   return -1;
    7989             : }
    7990             : 
    7991      230859 : bool BuildVectorSDNode::isConstant() const {
    7992     2627970 :   for (const SDValue &Op : op_values()) {
    7993      987843 :     unsigned Opc = Op.getOpcode();
    7994      987843 :     if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
    7995             :       return false;
    7996             :   }
    7997             :   return true;
    7998             : }
    7999             : 
    8000       47420 : bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
    8001             :   // Find the first non-undef value in the shuffle mask.
    8002             :   unsigned i, e;
    8003       47420 :   for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
    8004             :     /* search */;
    8005             : 
    8006             :   assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
    8007             : 
    8008             :   // Make sure all remaining elements are either undef or the same as the first
    8009             :   // non-undef value.
    8010      163844 :   for (int Idx = Mask[i]; i != e; ++i)
    8011      155523 :     if (Mask[i] >= 0 && Mask[i] != Idx)
    8012             :       return false;
    8013             :   return true;
    8014             : }
    8015             : 
    8016             : // \brief Returns the SDNode if it is a constant integer BuildVector
    8017             : // or constant integer.
    8018     3571765 : SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
    8019     3214395 :   if (isa<ConstantSDNode>(N))
    8020             :     return N.getNode();
    8021     3214395 :   if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
    8022             :     return N.getNode();
    8023             :   // Treat a GlobalAddress supporting constant offset folding as a
    8024             :   // constant integer.
    8025      407593 :   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
    8026      815171 :     if (GA->getOpcode() == ISD::GlobalAddress &&
    8027      407578 :         TLI->isOffsetFoldingLegal(GA))
    8028             :       return GA;
    8029             :   return nullptr;
    8030             : }
    8031             : 
    8032        5434 : SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
    8033        5208 :   if (isa<ConstantFPSDNode>(N))
    8034             :     return N.getNode();
    8035             : 
    8036        5208 :   if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
    8037           4 :     return N.getNode();
    8038             : 
    8039             :   return nullptr;
    8040             : }
    8041             : 
    8042             : #ifndef NDEBUG
    8043             : static void checkForCyclesHelper(const SDNode *N,
    8044             :                                  SmallPtrSetImpl<const SDNode*> &Visited,
    8045             :                                  SmallPtrSetImpl<const SDNode*> &Checked,
    8046             :                                  const llvm::SelectionDAG *DAG) {
    8047             :   // If this node has already been checked, don't check it again.
    8048             :   if (Checked.count(N))
    8049             :     return;
    8050             : 
    8051             :   // If a node has already been visited on this depth-first walk, reject it as
    8052             :   // a cycle.
    8053             :   if (!Visited.insert(N).second) {
    8054             :     errs() << "Detected cycle in SelectionDAG\n";
    8055             :     dbgs() << "Offending node:\n";
    8056             :     N->dumprFull(DAG); dbgs() << "\n";
    8057             :     abort();
    8058             :   }
    8059             : 
    8060             :   for (const SDValue &Op : N->op_values())
    8061             :     checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
    8062             : 
    8063             :   Checked.insert(N);
    8064             :   Visited.erase(N);
    8065             : }
    8066             : #endif
    8067             : 
    8068    62829162 : void llvm::checkForCycles(const llvm::SDNode *N,
    8069             :                           const llvm::SelectionDAG *DAG,
    8070             :                           bool force) {
    8071             : #ifndef NDEBUG
    8072             :   bool check = force;
    8073             : #ifdef EXPENSIVE_CHECKS
    8074             :   check = true;
    8075             : #endif  // EXPENSIVE_CHECKS
    8076             :   if (check) {
    8077             :     assert(N && "Checking nonexistent SDNode");
    8078             :     SmallPtrSet<const SDNode*, 32> visited;
    8079             :     SmallPtrSet<const SDNode*, 32> checked;
    8080             :     checkForCyclesHelper(N, visited, checked, DAG);
    8081             :   }
    8082             : #endif  // !NDEBUG
    8083    62829162 : }
    8084             : 
    8085     4427205 : void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
    8086     4427205 :   checkForCycles(DAG->getRoot().getNode(), DAG, force);
    8087     4427205 : }

Generated by: LCOV version 1.13