LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeDAG.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1925 2159 89.2 %
Date: 2018-02-18 16:14:26 Functions: 45 47 95.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the SelectionDAG::Legalize method.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/ADT/APFloat.h"
      15             : #include "llvm/ADT/APInt.h"
      16             : #include "llvm/ADT/ArrayRef.h"
      17             : #include "llvm/ADT/SetVector.h"
      18             : #include "llvm/ADT/SmallPtrSet.h"
      19             : #include "llvm/ADT/SmallSet.h"
      20             : #include "llvm/ADT/SmallVector.h"
      21             : #include "llvm/CodeGen/ISDOpcodes.h"
      22             : #include "llvm/CodeGen/MachineFunction.h"
      23             : #include "llvm/CodeGen/MachineJumpTableInfo.h"
      24             : #include "llvm/CodeGen/MachineMemOperand.h"
      25             : #include "llvm/CodeGen/MachineValueType.h"
      26             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      27             : #include "llvm/CodeGen/SelectionDAG.h"
      28             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      29             : #include "llvm/CodeGen/TargetFrameLowering.h"
      30             : #include "llvm/CodeGen/TargetLowering.h"
      31             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      32             : #include "llvm/CodeGen/ValueTypes.h"
      33             : #include "llvm/IR/CallingConv.h"
      34             : #include "llvm/IR/Constants.h"
      35             : #include "llvm/IR/DataLayout.h"
      36             : #include "llvm/IR/DerivedTypes.h"
      37             : #include "llvm/IR/Function.h"
      38             : #include "llvm/IR/Metadata.h"
      39             : #include "llvm/IR/Type.h"
      40             : #include "llvm/Support/Casting.h"
      41             : #include "llvm/Support/Compiler.h"
      42             : #include "llvm/Support/Debug.h"
      43             : #include "llvm/Support/ErrorHandling.h"
      44             : #include "llvm/Support/MathExtras.h"
      45             : #include "llvm/Support/raw_ostream.h"
      46             : #include "llvm/Target/TargetMachine.h"
      47             : #include "llvm/Target/TargetOptions.h"
      48             : #include <algorithm>
      49             : #include <cassert>
      50             : #include <cstdint>
      51             : #include <tuple>
      52             : #include <utility>
      53             : 
      54             : using namespace llvm;
      55             : 
      56             : #define DEBUG_TYPE "legalizedag"
      57             : 
      58             : namespace {
      59             : 
      60             : /// Keeps track of state when getting the sign of a floating-point value as an
      61             : /// integer.
      62          79 : struct FloatSignAsInt {
      63             :   EVT FloatVT;
      64             :   SDValue Chain;
      65             :   SDValue FloatPtr;
      66             :   SDValue IntPtr;
      67             :   MachinePointerInfo IntPointerInfo;
      68             :   MachinePointerInfo FloatPointerInfo;
      69             :   SDValue IntValue;
      70             :   APInt SignMask;
      71             :   uint8_t SignBit;
      72             : };
      73             : 
      74             : //===----------------------------------------------------------------------===//
      75             : /// This takes an arbitrary SelectionDAG as input and
      76             : /// hacks on it until the target machine can handle it.  This involves
      77             : /// eliminating value sizes the machine cannot handle (promoting small sizes to
      78             : /// large sizes or splitting up large values into small values) as well as
      79             : /// eliminating operations the machine cannot handle.
      80             : ///
      81             : /// This code also does a small amount of optimization and recognition of idioms
      82             : /// as part of its processing.  For example, if a target does not support a
      83             : /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
      84             : /// will attempt merge setcc and brc instructions into brcc's.
      85             : class SelectionDAGLegalize {
      86             :   const TargetMachine &TM;
      87             :   const TargetLowering &TLI;
      88             :   SelectionDAG &DAG;
      89             : 
      90             :   /// \brief The set of nodes which have already been legalized. We hold a
      91             :   /// reference to it in order to update as necessary on node deletion.
      92             :   SmallPtrSetImpl<SDNode *> &LegalizedNodes;
      93             : 
      94             :   /// \brief A set of all the nodes updated during legalization.
      95             :   SmallSetVector<SDNode *, 16> *UpdatedNodes;
      96             : 
      97         448 :   EVT getSetCCResultType(EVT VT) const {
      98         896 :     return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
      99             :   }
     100             : 
     101             :   // Libcall insertion helpers.
     102             : 
     103             : public:
     104             :   SelectionDAGLegalize(SelectionDAG &DAG,
     105             :                        SmallPtrSetImpl<SDNode *> &LegalizedNodes,
     106             :                        SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
     107    11764479 :       : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
     108    11764479 :         LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
     109             : 
     110             :   /// \brief Legalizes the given operation.
     111             :   void LegalizeOp(SDNode *Node);
     112             : 
     113             : private:
     114             :   SDValue OptimizeFloatStore(StoreSDNode *ST);
     115             : 
     116             :   void LegalizeLoadOps(SDNode *Node);
     117             :   void LegalizeStoreOps(SDNode *Node);
     118             : 
     119             :   /// Some targets cannot handle a variable
     120             :   /// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
     121             :   /// is necessary to spill the vector being inserted into to memory, perform
     122             :   /// the insert there, and then read the result back.
     123             :   SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
     124             :                                          const SDLoc &dl);
     125             :   SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
     126             :                                   const SDLoc &dl);
     127             : 
     128             :   /// Return a vector shuffle operation which
     129             :   /// performs the same shuffe in terms of order or result bytes, but on a type
     130             :   /// whose vector element type is narrower than the original shuffle type.
     131             :   /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
     132             :   SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
     133             :                                      SDValue N1, SDValue N2,
     134             :                                      ArrayRef<int> Mask) const;
     135             : 
     136             :   bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
     137             :                              bool &NeedInvert, const SDLoc &dl);
     138             : 
     139             :   SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
     140             :   SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
     141             :                         unsigned NumOps, bool isSigned, const SDLoc &dl);
     142             : 
     143             :   std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
     144             :                                                  SDNode *Node, bool isSigned);
     145             :   SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
     146             :                           RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
     147             :                           RTLIB::Libcall Call_F128,
     148             :                           RTLIB::Libcall Call_PPCF128);
     149             :   SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
     150             :                            RTLIB::Libcall Call_I8,
     151             :                            RTLIB::Libcall Call_I16,
     152             :                            RTLIB::Libcall Call_I32,
     153             :                            RTLIB::Libcall Call_I64,
     154             :                            RTLIB::Libcall Call_I128);
     155             :   void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
     156             :   void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
     157             : 
     158             :   SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
     159             :                            const SDLoc &dl);
     160             :   SDValue ExpandBUILD_VECTOR(SDNode *Node);
     161             :   SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
     162             :   void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
     163             :                                 SmallVectorImpl<SDValue> &Results);
     164             :   void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
     165             :                          SDValue Value) const;
     166             :   SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
     167             :                           SDValue NewIntValue) const;
     168             :   SDValue ExpandFCOPYSIGN(SDNode *Node) const;
     169             :   SDValue ExpandFABS(SDNode *Node) const;
     170             :   SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
     171             :                                const SDLoc &dl);
     172             :   SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
     173             :                                 const SDLoc &dl);
     174             :   SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
     175             :                                 const SDLoc &dl);
     176             : 
     177             :   SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
     178             :   SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
     179             :   SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
     180             : 
     181             :   SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
     182             :   SDValue ExpandInsertToVectorThroughStack(SDValue Op);
     183             :   SDValue ExpandVectorBuildThroughStack(SDNode* Node);
     184             : 
     185             :   SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
     186             :   SDValue ExpandConstant(ConstantSDNode *CP);
     187             : 
     188             :   // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
     189             :   bool ExpandNode(SDNode *Node);
     190             :   void ConvertNodeToLibcall(SDNode *Node);
     191             :   void PromoteNode(SDNode *Node);
     192             : 
     193             : public:
     194             :   // Node replacement helpers
     195             : 
     196     1115365 :   void ReplacedNode(SDNode *N) {
     197     1115365 :     LegalizedNodes.erase(N);
     198     1115365 :     if (UpdatedNodes)
     199        9473 :       UpdatedNodes->insert(N);
     200     1115365 :   }
     201             : 
     202         124 :   void ReplaceNode(SDNode *Old, SDNode *New) {
     203             :     DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
     204             :           dbgs() << "     with:      "; New->dump(&DAG));
     205             : 
     206             :     assert(Old->getNumValues() == New->getNumValues() &&
     207             :            "Replacing one node with another that produces a different number "
     208             :            "of values!");
     209         124 :     DAG.ReplaceAllUsesWith(Old, New);
     210         124 :     if (UpdatedNodes)
     211           1 :       UpdatedNodes->insert(New);
     212         124 :     ReplacedNode(Old);
     213         124 :   }
     214             : 
     215      701558 :   void ReplaceNode(SDValue Old, SDValue New) {
     216             :     DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
     217             :           dbgs() << "     with:      "; New->dump(&DAG));
     218             : 
     219      701558 :     DAG.ReplaceAllUsesWith(Old, New);
     220      701558 :     if (UpdatedNodes)
     221        3645 :       UpdatedNodes->insert(New.getNode());
     222      701558 :     ReplacedNode(Old.getNode());
     223      701558 :   }
     224             : 
     225      334970 :   void ReplaceNode(SDNode *Old, const SDValue *New) {
     226             :     DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
     227             : 
     228      334970 :     DAG.ReplaceAllUsesWith(Old, New);
     229     1583216 :     for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
     230             :       DEBUG(dbgs() << (i == 0 ? "     with:      "
     231             :                               : "      and:      ");
     232             :             New[i]->dump(&DAG));
     233      624123 :       if (UpdatedNodes)
     234        3456 :         UpdatedNodes->insert(New[i].getNode());
     235             :     }
     236      334970 :     ReplacedNode(Old);
     237      334970 :   }
     238             : };
     239             : 
     240             : } // end anonymous namespace
     241             : 
     242             : /// Return a vector shuffle operation which
     243             : /// performs the same shuffe in terms of order or result bytes, but on a type
     244             : /// whose vector element type is narrower than the original shuffle type.
     245             : /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
     246         507 : SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
     247             :     EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
     248             :     ArrayRef<int> Mask) const {
     249         507 :   unsigned NumMaskElts = VT.getVectorNumElements();
     250         507 :   unsigned NumDestElts = NVT.getVectorNumElements();
     251         507 :   unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
     252             : 
     253             :   assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
     254             : 
     255         507 :   if (NumEltsGrowth == 1)
     256           0 :     return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
     257             : 
     258             :   SmallVector<int, 8> NewMask;
     259        6315 :   for (unsigned i = 0; i != NumMaskElts; ++i) {
     260        5808 :     int Idx = Mask[i];
     261       19800 :     for (unsigned j = 0; j != NumEltsGrowth; ++j) {
     262        8448 :       if (Idx < 0)
     263         828 :         NewMask.push_back(-1);
     264             :       else
     265        7620 :         NewMask.push_back(Idx * NumEltsGrowth + j);
     266             :     }
     267             :   }
     268             :   assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
     269             :   assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
     270        1014 :   return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
     271             : }
     272             : 
     273             : /// Expands the ConstantFP node to an integer constant or
     274             : /// a load from the constant pool.
     275             : SDValue
     276        2936 : SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
     277             :   bool Extend = false;
     278             :   SDLoc dl(CFP);
     279             : 
     280             :   // If a FP immediate is precise when represented as a float and if the
     281             :   // target can do an extending load from float to double, we put it into
     282             :   // the constant pool as a float, even if it's is statically typed as a
     283             :   // double.  This shrinks FP constants and canonicalizes them for targets where
     284             :   // an FP extending load is the same cost as a normal load (such as on the x87
     285             :   // fp stack or PPC FP unit).
     286        2936 :   EVT VT = CFP->getValueType(0);
     287             :   ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
     288        2936 :   if (!UseCP) {
     289             :     assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
     290           0 :     return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
     291           0 :                            (VT == MVT::f64) ? MVT::i64 : MVT::i32);
     292             :   }
     293             : 
     294             :   APFloat APF = CFP->getValueAPF();
     295        2936 :   EVT OrigVT = VT;
     296        2936 :   EVT SVT = VT;
     297             : 
     298             :   // We don't want to shrink SNaNs. Converting the SNaN back to its real type
     299             :   // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
     300        2936 :   if (!APF.isSignaling()) {
     301             :     while (SVT != MVT::f32 && SVT != MVT::f16) {
     302        1582 :       SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
     303        2777 :       if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
     304             :           // Only do this if the target has a native EXTLOAD instruction from
     305             :           // smaller type.
     306        2492 :           TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
     307         910 :           TLI.ShouldShrinkFPConstant(OrigVT)) {
     308         516 :         Type *SType = SVT.getTypeForEVT(*DAG.getContext());
     309         516 :         LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
     310         516 :         VT = SVT;
     311             :         Extend = true;
     312             :       }
     313             :     }
     314             :   }
     315             : 
     316             :   SDValue CPIdx =
     317        5872 :       DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
     318             :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
     319        2936 :   if (Extend) {
     320         433 :     SDValue Result = DAG.getExtLoad(
     321         433 :         ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
     322             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
     323         866 :         Alignment);
     324         433 :     return Result;
     325             :   }
     326        2503 :   SDValue Result = DAG.getLoad(
     327        2503 :       OrigVT, dl, DAG.getEntryNode(), CPIdx,
     328        5006 :       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
     329        2503 :   return Result;
     330             : }
     331             : 
     332             : /// Expands the Constant node to a load from the constant pool.
     333           0 : SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
     334             :   SDLoc dl(CP);
     335           0 :   EVT VT = CP->getValueType(0);
     336           0 :   SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
     337           0 :                                       TLI.getPointerTy(DAG.getDataLayout()));
     338             :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
     339           0 :   SDValue Result = DAG.getLoad(
     340           0 :       VT, dl, DAG.getEntryNode(), CPIdx,
     341           0 :       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
     342           0 :   return Result;
     343             : }
     344             : 
     345             : /// Some target cannot handle a variable insertion index for the
     346             : /// INSERT_VECTOR_ELT instruction.  In this case, it
     347             : /// is necessary to spill the vector being inserted into to memory, perform
     348             : /// the insert there, and then read the result back.
     349          20 : SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
     350             :                                                              SDValue Val,
     351             :                                                              SDValue Idx,
     352             :                                                              const SDLoc &dl) {
     353          20 :   SDValue Tmp1 = Vec;
     354          20 :   SDValue Tmp2 = Val;
     355          20 :   SDValue Tmp3 = Idx;
     356             : 
     357             :   // If the target doesn't support this, we have to spill the input vector
     358             :   // to a temporary stack slot, update the element, then reload it.  This is
     359             :   // badness.  We could also load the value into a vector register (either
     360             :   // with a "move to register" or "extload into register" instruction, then
     361             :   // permute it into place, if the idx is a constant and if the idx is
     362             :   // supported by the target.
     363          20 :   EVT VT    = Tmp1.getValueType();
     364          20 :   EVT EltVT = VT.getVectorElementType();
     365          20 :   SDValue StackPtr = DAG.CreateStackTemporary(VT);
     366             : 
     367             :   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
     368             : 
     369             :   // Store the vector.
     370          20 :   SDValue Ch = DAG.getStore(
     371          20 :       DAG.getEntryNode(), dl, Tmp1, StackPtr,
     372          40 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
     373             : 
     374          20 :   SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
     375             : 
     376             :   // Store the scalar value.
     377          40 :   Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
     378             :   // Load the updated vector.
     379          20 :   return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
     380          20 :                                                DAG.getMachineFunction(), SPFI));
     381             : }
     382             : 
     383         589 : SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
     384             :                                                       SDValue Idx,
     385             :                                                       const SDLoc &dl) {
     386             :   if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
     387             :     // SCALAR_TO_VECTOR requires that the type of the value being inserted
     388             :     // match the element type of the vector being created, except for
     389             :     // integers in which case the inserted value can be over width.
     390         569 :     EVT EltVT = Vec.getValueType().getVectorElementType();
     391           1 :     if (Val.getValueType() == EltVT ||
     392           2 :         (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
     393         569 :       SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
     394         569 :                                   Vec.getValueType(), Val);
     395             : 
     396         569 :       unsigned NumElts = Vec.getValueType().getVectorNumElements();
     397             :       // We generate a shuffle of InVec and ScVec, so the shuffle mask
     398             :       // should be 0,1,2,3,4,5... with the appropriate element replaced with
     399             :       // elt 0 of the RHS.
     400             :       SmallVector<int, 8> ShufOps;
     401        5177 :       for (unsigned i = 0; i != NumElts; ++i)
     402        4608 :         ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
     403             : 
     404        1138 :       return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
     405             :     }
     406             :   }
     407          20 :   return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
     408             : }
     409             : 
     410     1475608 : SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
     411             :   DEBUG(dbgs() << "Optimizing float store operations\n");
     412             :   // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
     413             :   // FIXME: We shouldn't do this for TargetConstantFP's.
     414             :   // FIXME: move this to the DAG Combiner!  Note that we can't regress due
     415             :   // to phase ordering between legalized code and the dag combiner.  This
     416             :   // probably means that we need to integrate dag combiner and legalizer
     417             :   // together.
     418             :   // We generally can't do this one for long doubles.
     419     1475608 :   SDValue Chain = ST->getChain();
     420     1475608 :   SDValue Ptr = ST->getBasePtr();
     421             :   unsigned Alignment = ST->getAlignment();
     422             :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
     423             :   AAMDNodes AAInfo = ST->getAAInfo();
     424             :   SDLoc dl(ST);
     425             :   if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
     426             :     if (CFP->getValueType(0) == MVT::f32 &&
     427           1 :         TLI.isTypeLegal(MVT::i32)) {
     428           1 :       SDValue Con = DAG.getConstant(CFP->getValueAPF().
     429           3 :                                       bitcastToAPInt().zextOrTrunc(32),
     430           2 :                                     SDLoc(CFP), MVT::i32);
     431           1 :       return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
     432           2 :                           MMOFlags, AAInfo);
     433             :     }
     434             : 
     435             :     if (CFP->getValueType(0) == MVT::f64) {
     436             :       // If this target supports 64-bit registers, do a single 64-bit store.
     437           8 :       if (TLI.isTypeLegal(MVT::i64)) {
     438           0 :         SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
     439           0 :                                       zextOrTrunc(64), SDLoc(CFP), MVT::i64);
     440           0 :         return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
     441           0 :                             Alignment, MMOFlags, AAInfo);
     442             :       }
     443             : 
     444           8 :       if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
     445             :         // Otherwise, if the target supports 32-bit registers, use 2 32-bit
     446             :         // stores.  If the target supports neither 32- nor 64-bits, this
     447             :         // xform is certainly not worth it.
     448           0 :         const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
     449           0 :         SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
     450           0 :         SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
     451           0 :         if (DAG.getDataLayout().isBigEndian())
     452             :           std::swap(Lo, Hi);
     453             : 
     454           0 :         Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
     455           0 :                           MMOFlags, AAInfo);
     456           0 :         Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     457           0 :                           DAG.getConstant(4, dl, Ptr.getValueType()));
     458           0 :         Hi = DAG.getStore(Chain, dl, Hi, Ptr,
     459             :                           ST->getPointerInfo().getWithOffset(4),
     460           0 :                           MinAlign(Alignment, 4U), MMOFlags, AAInfo);
     461             : 
     462           0 :         return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
     463             :       }
     464             :     }
     465             :   }
     466     1475607 :   return SDValue(nullptr, 0);
     467             : }
     468             : 
     469     1503261 : void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
     470             :   StoreSDNode *ST = cast<StoreSDNode>(Node);
     471     1503261 :   SDValue Chain = ST->getChain();
     472     1503261 :   SDValue Ptr = ST->getBasePtr();
     473             :   SDLoc dl(Node);
     474             : 
     475             :   unsigned Alignment = ST->getAlignment();
     476             :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
     477             :   AAMDNodes AAInfo = ST->getAAInfo();
     478             : 
     479     1503261 :   if (!ST->isTruncatingStore()) {
     480             :     DEBUG(dbgs() << "Legalizing store operation\n");
     481     1475608 :     if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
     482           1 :       ReplaceNode(ST, OptStore);
     483           1 :       return;
     484             :     }
     485             : 
     486     1475607 :     SDValue Value = ST->getValue();
     487             :     MVT VT = Value.getSimpleValueType();
     488     2951214 :     switch (TLI.getOperationAction(ISD::STORE, VT)) {
     489           0 :     default: llvm_unreachable("This action is not supported yet!");
     490     1354309 :     case TargetLowering::Legal: {
     491             :       // If this is an unaligned store and the target doesn't support it,
     492             :       // expand it.
     493     1354309 :       EVT MemVT = ST->getMemoryVT();
     494             :       unsigned AS = ST->getAddressSpace();
     495             :       unsigned Align = ST->getAlignment();
     496     1354309 :       const DataLayout &DL = DAG.getDataLayout();
     497     1354309 :       if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     498             :         DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
     499         230 :         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
     500         230 :         ReplaceNode(SDValue(ST, 0), Result);
     501             :       } else
     502             :         DEBUG(dbgs() << "Legal store\n");
     503             :       break;
     504             :     }
     505      111602 :     case TargetLowering::Custom: {
     506             :       DEBUG(dbgs() << "Trying custom lowering\n");
     507      223204 :       SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
     508      111602 :       if (Res && Res != SDValue(Node, 0))
     509       14056 :         ReplaceNode(SDValue(Node, 0), Res);
     510             :       return;
     511             :     }
     512        9696 :     case TargetLowering::Promote: {
     513        9696 :       MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
     514             :       assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
     515             :              "Can only promote stores to same size type");
     516       19392 :       Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
     517             :       SDValue Result =
     518        9696 :           DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     519       19392 :                        Alignment, MMOFlags, AAInfo);
     520        9696 :       ReplaceNode(SDValue(Node, 0), Result);
     521             :       break;
     522             :     }
     523             :     }
     524             :     return;
     525             :   }
     526             : 
     527             :   DEBUG(dbgs() << "Legalizing truncating store operations\n");
     528       27653 :   SDValue Value = ST->getValue();
     529       27653 :   EVT StVT = ST->getMemoryVT();
     530       27653 :   unsigned StWidth = StVT.getSizeInBits();
     531       27653 :   auto &DL = DAG.getDataLayout();
     532             : 
     533       27653 :   if (StWidth != StVT.getStoreSizeInBits()) {
     534             :     // Promote to a byte-sized store with upper bits zero if not
     535             :     // storing an integral number of bytes.  For example, promote
     536             :     // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
     537        1781 :     EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
     538        1781 :                                 StVT.getStoreSizeInBits());
     539        1781 :     Value = DAG.getZeroExtendInReg(Value, dl, StVT);
     540             :     SDValue Result =
     541        1781 :         DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
     542        3562 :                           Alignment, MMOFlags, AAInfo);
     543        1781 :     ReplaceNode(SDValue(Node, 0), Result);
     544       25872 :   } else if (StWidth & (StWidth - 1)) {
     545             :     // If not storing a power-of-2 number of bits, expand as two stores.
     546             :     assert(!StVT.isVector() && "Unsupported truncstore!");
     547         707 :     unsigned RoundWidth = 1 << Log2_32(StWidth);
     548             :     assert(RoundWidth < StWidth);
     549         707 :     unsigned ExtraWidth = StWidth - RoundWidth;
     550             :     assert(ExtraWidth < RoundWidth);
     551             :     assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
     552             :            "Store size not an integral number of bytes!");
     553         707 :     EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
     554         707 :     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
     555         707 :     SDValue Lo, Hi;
     556             :     unsigned IncrementSize;
     557             : 
     558         707 :     if (DL.isLittleEndian()) {
     559             :       // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
     560             :       // Store the bottom RoundWidth bits.
     561        1390 :       Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     562         695 :                              RoundVT, Alignment, MMOFlags, AAInfo);
     563             : 
     564             :       // Store the remaining ExtraWidth bits.
     565         695 :       IncrementSize = RoundWidth / 8;
     566        1390 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     567             :                         DAG.getConstant(IncrementSize, dl,
     568        1390 :                                         Ptr.getValueType()));
     569        1390 :       Hi = DAG.getNode(
     570             :           ISD::SRL, dl, Value.getValueType(), Value,
     571             :           DAG.getConstant(RoundWidth, dl,
     572        2085 :                           TLI.getShiftAmountTy(Value.getValueType(), DL)));
     573        1390 :       Hi = DAG.getTruncStore(
     574             :           Chain, dl, Hi, Ptr,
     575             :           ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
     576        1390 :           MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
     577             :     } else {
     578             :       // Big endian - avoid unaligned stores.
     579             :       // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
     580             :       // Store the top RoundWidth bits.
     581          24 :       Hi = DAG.getNode(
     582             :           ISD::SRL, dl, Value.getValueType(), Value,
     583             :           DAG.getConstant(ExtraWidth, dl,
     584          36 :                           TLI.getShiftAmountTy(Value.getValueType(), DL)));
     585          24 :       Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
     586          12 :                              RoundVT, Alignment, MMOFlags, AAInfo);
     587             : 
     588             :       // Store the remaining ExtraWidth bits.
     589          12 :       IncrementSize = RoundWidth / 8;
     590          24 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     591             :                         DAG.getConstant(IncrementSize, dl,
     592          24 :                                         Ptr.getValueType()));
     593          24 :       Lo = DAG.getTruncStore(
     594             :           Chain, dl, Value, Ptr,
     595             :           ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
     596          24 :           MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
     597             :     }
     598             : 
     599             :     // The order of the stores doesn't matter.
     600        1414 :     SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
     601         707 :     ReplaceNode(SDValue(Node, 0), Result);
     602             :   } else {
     603       50330 :     switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
     604           0 :     default: llvm_unreachable("This action is not supported yet!");
     605       21935 :     case TargetLowering::Legal: {
     606       21935 :       EVT MemVT = ST->getMemoryVT();
     607             :       unsigned AS = ST->getAddressSpace();
     608             :       unsigned Align = ST->getAlignment();
     609             :       // If this is an unaligned store and the target doesn't support it,
     610             :       // expand it.
     611       21935 :       if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     612         449 :         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
     613         449 :         ReplaceNode(SDValue(ST, 0), Result);
     614             :       }
     615             :       break;
     616             :     }
     617        1659 :     case TargetLowering::Custom: {
     618        3318 :       SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
     619        1659 :       if (Res && Res != SDValue(Node, 0))
     620        1497 :         ReplaceNode(SDValue(Node, 0), Res);
     621             :       return;
     622             :     }
     623        1571 :     case TargetLowering::Expand:
     624             :       assert(!StVT.isVector() &&
     625             :              "Vector Stores are handled in LegalizeVectorOps");
     626             : 
     627        1571 :       SDValue Result;
     628             : 
     629             :       // TRUNCSTORE:i16 i32 -> STORE i16
     630             :       if (TLI.isTypeLegal(StVT)) {
     631        3118 :         Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
     632        3118 :         Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     633        1559 :                               Alignment, MMOFlags, AAInfo);
     634             :       } else {
     635             :         // The in-memory type isn't legal. Truncate to the type it would promote
     636             :         // to, and then do a truncstore.
     637          24 :         Value = DAG.getNode(ISD::TRUNCATE, dl,
     638             :                             TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
     639          24 :                             Value);
     640          24 :         Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     641          12 :                                    StVT, Alignment, MMOFlags, AAInfo);
     642             :       }
     643             : 
     644        1571 :       ReplaceNode(SDValue(Node, 0), Result);
     645        1571 :       break;
     646             :     }
     647             :   }
     648             : }
     649             : 
     650     1157534 : void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
     651             :   LoadSDNode *LD = cast<LoadSDNode>(Node);
     652     1157534 :   SDValue Chain = LD->getChain();  // The chain.
     653     1157534 :   SDValue Ptr = LD->getBasePtr();  // The base pointer.
     654     1157534 :   SDValue Value;                   // The value returned by the load op.
     655             :   SDLoc dl(Node);
     656             : 
     657             :   ISD::LoadExtType ExtType = LD->getExtensionType();
     658     1157534 :   if (ExtType == ISD::NON_EXTLOAD) {
     659             :     DEBUG(dbgs() << "Legalizing non-extending load operation\n");
     660             :     MVT VT = Node->getSimpleValueType(0);
     661             :     SDValue RVal = SDValue(Node, 0);
     662             :     SDValue RChain = SDValue(Node, 1);
     663             : 
     664     2191718 :     switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
     665           0 :     default: llvm_unreachable("This action is not supported yet!");
     666      904709 :     case TargetLowering::Legal: {
     667      904709 :       EVT MemVT = LD->getMemoryVT();
     668             :       unsigned AS = LD->getAddressSpace();
     669             :       unsigned Align = LD->getAlignment();
     670      904709 :       const DataLayout &DL = DAG.getDataLayout();
     671             :       // If this is an unaligned load and the target doesn't support it,
     672             :       // expand it.
     673      904709 :       if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     674         472 :         std::tie(RVal, RChain) =  TLI.expandUnalignedLoad(LD, DAG);
     675             :       }
     676             :       break;
     677             :     }
     678      132829 :     case TargetLowering::Custom:
     679      132829 :       if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
     680       18210 :         RVal = Res;
     681       18210 :         RChain = Res.getValue(1);
     682      132829 :       }
     683      132829 :       break;
     684             : 
     685       58321 :     case TargetLowering::Promote: {
     686       58321 :       MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
     687             :       assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
     688             :              "Can only promote loads to same size type");
     689             : 
     690      116642 :       SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
     691      116642 :       RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
     692       58321 :       RChain = Res.getValue(1);
     693             :       break;
     694             :     }
     695             :     }
     696     1095859 :     if (RChain.getNode() != Node) {
     697             :       assert(RVal.getNode() != Node && "Load must be completely replaced");
     698      145492 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
     699      145492 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
     700       72746 :       if (UpdatedNodes) {
     701        2630 :         UpdatedNodes->insert(RVal.getNode());
     702        2630 :         UpdatedNodes->insert(RChain.getNode());
     703             :       }
     704       72746 :       ReplacedNode(Node);
     705             :     }
     706             :     return;
     707             :   }
     708             : 
     709             :   DEBUG(dbgs() << "Legalizing extending load operation\n");
     710       61675 :   EVT SrcVT = LD->getMemoryVT();
     711       61675 :   unsigned SrcWidth = SrcVT.getSizeInBits();
     712             :   unsigned Alignment = LD->getAlignment();
     713             :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
     714             :   AAMDNodes AAInfo = LD->getAAInfo();
     715             : 
     716       61675 :   if (SrcWidth != SrcVT.getStoreSizeInBits() &&
     717             :       // Some targets pretend to have an i1 loading operation, and actually
     718             :       // load an i8.  This trick is correct for ZEXTLOAD because the top 7
     719             :       // bits are guaranteed to be zero; it helps the optimizers understand
     720             :       // that these bits are zero.  It is also useful for EXTLOAD, since it
     721             :       // tells the optimizers that those bits are undefined.  It would be
     722             :       // nice to have an effective generic way of getting these benefits...
     723             :       // Until such a way is found, don't insist on promoting i1 here.
     724         557 :       (SrcVT != MVT::i1 ||
     725         557 :        TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
     726             :          TargetLowering::Promote)) {
     727             :     // Promote to a byte-sized load if not loading an integral number of
     728             :     // bytes.  For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
     729             :     unsigned NewWidth = SrcVT.getStoreSizeInBits();
     730         582 :     EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
     731             :     SDValue Ch;
     732             : 
     733             :     // The extra bits are guaranteed to be zero, since we stored them that
     734             :     // way.  A zext load from NVT thus automatically gives zext from SrcVT.
     735             : 
     736             :     ISD::LoadExtType NewExtType =
     737         582 :       ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
     738             : 
     739             :     SDValue Result =
     740         582 :         DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
     741        1164 :                        LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
     742             : 
     743             :     Ch = Result.getValue(1); // The chain.
     744             : 
     745         582 :     if (ExtType == ISD::SEXTLOAD)
     746             :       // Having the top bits zero doesn't help when sign extending.
     747         180 :       Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
     748             :                            Result.getValueType(),
     749         180 :                            Result, DAG.getValueType(SrcVT));
     750         492 :     else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
     751             :       // All the top bits are guaranteed to be zero - inform the optimizers.
     752         600 :       Result = DAG.getNode(ISD::AssertZext, dl,
     753             :                            Result.getValueType(), Result,
     754         600 :                            DAG.getValueType(SrcVT));
     755             : 
     756         582 :     Value = Result;
     757         582 :     Chain = Ch;
     758       61093 :   } else if (SrcWidth & (SrcWidth - 1)) {
     759             :     // If not loading a power-of-2 number of bits, expand as two loads.
     760             :     assert(!SrcVT.isVector() && "Unsupported extload!");
     761         254 :     unsigned RoundWidth = 1 << Log2_32(SrcWidth);
     762             :     assert(RoundWidth < SrcWidth);
     763         254 :     unsigned ExtraWidth = SrcWidth - RoundWidth;
     764             :     assert(ExtraWidth < RoundWidth);
     765             :     assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
     766             :            "Load size not an integral number of bytes!");
     767         254 :     EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
     768         254 :     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
     769         254 :     SDValue Lo, Hi, Ch;
     770             :     unsigned IncrementSize;
     771         254 :     auto &DL = DAG.getDataLayout();
     772             : 
     773         254 :     if (DL.isLittleEndian()) {
     774             :       // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
     775             :       // Load the bottom RoundWidth bits.
     776         458 :       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
     777             :                           LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
     778         229 :                           AAInfo);
     779             : 
     780             :       // Load the remaining ExtraWidth bits.
     781         229 :       IncrementSize = RoundWidth / 8;
     782         458 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     783             :                          DAG.getConstant(IncrementSize, dl,
     784         458 :                                          Ptr.getValueType()));
     785         458 :       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
     786             :                           LD->getPointerInfo().getWithOffset(IncrementSize),
     787             :                           ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
     788         458 :                           AAInfo);
     789             : 
     790             :       // Build a factor node to remember that this load is independent of
     791             :       // the other one.
     792         458 :       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
     793         229 :                        Hi.getValue(1));
     794             : 
     795             :       // Move the top bits to the right place.
     796         458 :       Hi = DAG.getNode(
     797             :           ISD::SHL, dl, Hi.getValueType(), Hi,
     798             :           DAG.getConstant(RoundWidth, dl,
     799         687 :                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
     800             : 
     801             :       // Join the hi and lo parts.
     802         458 :       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
     803             :     } else {
     804             :       // Big endian - avoid unaligned loads.
     805             :       // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
     806             :       // Load the top RoundWidth bits.
     807          50 :       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
     808             :                           LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
     809          25 :                           AAInfo);
     810             : 
     811             :       // Load the remaining ExtraWidth bits.
     812          25 :       IncrementSize = RoundWidth / 8;
     813          50 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     814             :                          DAG.getConstant(IncrementSize, dl,
     815          50 :                                          Ptr.getValueType()));
     816          50 :       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
     817             :                           LD->getPointerInfo().getWithOffset(IncrementSize),
     818             :                           ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
     819          50 :                           AAInfo);
     820             : 
     821             :       // Build a factor node to remember that this load is independent of
     822             :       // the other one.
     823          50 :       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
     824          25 :                        Hi.getValue(1));
     825             : 
     826             :       // Move the top bits to the right place.
     827          50 :       Hi = DAG.getNode(
     828             :           ISD::SHL, dl, Hi.getValueType(), Hi,
     829             :           DAG.getConstant(ExtraWidth, dl,
     830          75 :                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
     831             : 
     832             :       // Join the hi and lo parts.
     833          50 :       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
     834             :     }
     835             : 
     836         254 :     Chain = Ch;
     837             :   } else {
     838             :     bool isCustom = false;
     839      121678 :     switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
     840             :                                  SrcVT.getSimpleVT())) {
     841           0 :     default: llvm_unreachable("This action is not supported yet!");
     842        5979 :     case TargetLowering::Custom:
     843             :       isCustom = true;
     844             :       LLVM_FALLTHROUGH;
     845       60504 :     case TargetLowering::Legal:
     846       60504 :       Value = SDValue(Node, 0);
     847       60504 :       Chain = SDValue(Node, 1);
     848             : 
     849       60504 :       if (isCustom) {
     850       11958 :         if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
     851        4353 :           Value = Res;
     852        4353 :           Chain = Res.getValue(1);
     853             :         }
     854             :       } else {
     855             :         // If this is an unaligned load and the target doesn't support it,
     856             :         // expand it.
     857       54525 :         EVT MemVT = LD->getMemoryVT();
     858             :         unsigned AS = LD->getAddressSpace();
     859             :         unsigned Align = LD->getAlignment();
     860       54525 :         const DataLayout &DL = DAG.getDataLayout();
     861       54525 :         if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     862         912 :           std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
     863             :         }
     864             :       }
     865             :       break;
     866             : 
     867         335 :     case TargetLowering::Expand: {
     868         335 :       EVT DestVT = Node->getValueType(0);
     869         335 :       if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
     870             :         // If the source type is not legal, see if there is a legal extload to
     871             :         // an intermediate type that we can then extend further.
     872             :         EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
     873          45 :         if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
     874             :             TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
     875             :           // If we are loading a legal type, this is a non-extload followed by a
     876             :           // full extend.
     877             :           ISD::LoadExtType MidExtType =
     878           0 :               (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
     879             : 
     880         315 :           SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
     881         315 :                                         SrcVT, LD->getMemOperand());
     882             :           unsigned ExtendOp =
     883         315 :               ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
     884         630 :           Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
     885         315 :           Chain = Load.getValue(1);
     886             :           break;
     887             :         }
     888             : 
     889             :         // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
     890             :         // normal undefined upper bits behavior to allow using an in-reg extend
     891             :         // with the illegal FP type, so load as an integer and do the
     892             :         // from-integer conversion.
     893           0 :         if (SrcVT.getScalarType() == MVT::f16) {
     894           0 :           EVT ISrcVT = SrcVT.changeTypeToInteger();
     895           0 :           EVT IDestVT = DestVT.changeTypeToInteger();
     896           0 :           EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
     897             : 
     898           0 :           SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
     899             :                                           Chain, Ptr, ISrcVT,
     900           0 :                                           LD->getMemOperand());
     901           0 :           Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
     902           0 :           Chain = Result.getValue(1);
     903             :           break;
     904             :         }
     905             :       }
     906             : 
     907             :       assert(!SrcVT.isVector() &&
     908             :              "Vector Loads are handled in LegalizeVectorOps");
     909             : 
     910             :       // FIXME: This does not work for vectors on most targets.  Sign-
     911             :       // and zero-extend operations are currently folded into extending
     912             :       // loads, whether they are legal or not, and then we end up here
     913             :       // without any support for legalizing them.
     914             :       assert(ExtType != ISD::EXTLOAD &&
     915             :              "EXTLOAD should always be supported!");
     916             :       // Turn the unsupported load into an EXTLOAD followed by an
     917             :       // explicit zero/sign extend inreg.
     918          20 :       SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
     919             :                                       Node->getValueType(0),
     920             :                                       Chain, Ptr, SrcVT,
     921          20 :                                       LD->getMemOperand());
     922             :       SDValue ValRes;
     923          20 :       if (ExtType == ISD::SEXTLOAD)
     924          38 :         ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
     925             :                              Result.getValueType(),
     926          38 :                              Result, DAG.getValueType(SrcVT));
     927             :       else
     928           1 :         ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
     929          20 :       Value = ValRes;
     930          20 :       Chain = Result.getValue(1);
     931          20 :       break;
     932             :     }
     933             :     }
     934             :   }
     935             : 
     936             :   // Since loads produce two values, make sure to remember that we legalized
     937             :   // both of them.
     938       61675 :   if (Chain.getNode() != Node) {
     939             :     assert(Value.getNode() != Node && "Load must be completely replaced");
     940       11932 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
     941       11932 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
     942        5966 :     if (UpdatedNodes) {
     943           1 :       UpdatedNodes->insert(Value.getNode());
     944           1 :       UpdatedNodes->insert(Chain.getNode());
     945             :     }
     946        5966 :     ReplacedNode(Node);
     947             :   }
     948             : }
     949             : 
     950             : static TargetLowering::LegalizeAction
     951          34 : getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) {
     952             :   unsigned EqOpc;
     953          34 :   switch (Opcode) {
     954           0 :     default: llvm_unreachable("Unexpected FP pseudo-opcode");
     955             :     case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
     956           2 :     case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
     957           2 :     case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
     958           6 :     case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
     959           2 :     case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
     960           2 :     case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
     961           2 :     case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
     962           2 :     case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
     963           2 :     case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
     964           2 :     case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
     965           2 :     case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
     966           3 :     case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
     967           3 :     case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
     968             :   }
     969             : 
     970             :   auto Action = TLI.getOperationAction(EqOpc, VT);
     971             : 
     972             :   // We don't currently handle Custom or Promote for strict FP pseudo-ops.
     973             :   // For now, we just expand for those cases.
     974          34 :   if (Action != TargetLowering::Legal)
     975             :     Action = TargetLowering::Expand;
     976             : 
     977          34 :   return Action;
     978             : }
     979             : 
     980             : /// Return a legal replacement for the given operation, with all legal operands.
     981    23037584 : void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     982             :   DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
     983             : 
     984             :   // Allow illegal target nodes and illegal registers.
     985    23037584 :   if (Node->getOpcode() == ISD::TargetConstant ||
     986             :       Node->getOpcode() == ISD::Register)
     987             :     return;
     988             : 
     989             : #ifndef NDEBUG
     990             :   for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
     991             :     assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
     992             :               TargetLowering::TypeLegal ||
     993             :             TLI.isTypeLegal(Node->getValueType(i))) &&
     994             :            "Unexpected illegal type!");
     995             : 
     996             :   for (const SDValue &Op : Node->op_values())
     997             :     assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
     998             :               TargetLowering::TypeLegal ||
     999             :             TLI.isTypeLegal(Op.getValueType()) ||
    1000             :             Op.getOpcode() == ISD::TargetConstant ||
    1001             :             Op.getOpcode() == ISD::Register) &&
    1002             :             "Unexpected illegal type!");
    1003             : #endif
    1004             : 
    1005             :   // Figure out the correct action; the way to query this varies by opcode
    1006             :   TargetLowering::LegalizeAction Action = TargetLowering::Legal;
    1007             :   bool SimpleFinishLegalizing = true;
    1008    20395937 :   switch (Node->getOpcode()) {
    1009       58292 :   case ISD::INTRINSIC_W_CHAIN:
    1010             :   case ISD::INTRINSIC_WO_CHAIN:
    1011             :   case ISD::INTRINSIC_VOID:
    1012             :   case ISD::STACKSAVE:
    1013       58292 :     Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
    1014             :     break;
    1015           4 :   case ISD::GET_DYNAMIC_AREA_OFFSET:
    1016           4 :     Action = TLI.getOperationAction(Node->getOpcode(),
    1017             :                                     Node->getValueType(0));
    1018             :     break;
    1019         192 :   case ISD::VAARG:
    1020         192 :     Action = TLI.getOperationAction(Node->getOpcode(),
    1021             :                                     Node->getValueType(0));
    1022         192 :     if (Action != TargetLowering::Promote)
    1023             :       Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
    1024             :     break;
    1025      216532 :   case ISD::FP_TO_FP16:
    1026             :   case ISD::SINT_TO_FP:
    1027             :   case ISD::UINT_TO_FP:
    1028             :   case ISD::EXTRACT_VECTOR_ELT:
    1029      216532 :     Action = TLI.getOperationAction(Node->getOpcode(),
    1030             :                                     Node->getOperand(0).getValueType());
    1031             :     break;
    1032       26210 :   case ISD::FP_ROUND_INREG:
    1033             :   case ISD::SIGN_EXTEND_INREG: {
    1034             :     EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
    1035       26210 :     Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
    1036             :     break;
    1037             :   }
    1038        1470 :   case ISD::ATOMIC_STORE:
    1039        1470 :     Action = TLI.getOperationAction(Node->getOpcode(),
    1040             :                                     Node->getOperand(2).getValueType());
    1041             :     break;
    1042       72838 :   case ISD::SELECT_CC:
    1043             :   case ISD::SETCC:
    1044             :   case ISD::BR_CC: {
    1045       72838 :     unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
    1046             :                          Node->getOpcode() == ISD::SETCC ? 2 :
    1047             :                          Node->getOpcode() == ISD::SETCCE ? 3 : 1;
    1048       72838 :     unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
    1049             :     MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
    1050             :     ISD::CondCode CCCode =
    1051             :         cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
    1052       72838 :     Action = TLI.getCondCodeAction(CCCode, OpVT);
    1053       72838 :     if (Action == TargetLowering::Legal) {
    1054       72245 :       if (Node->getOpcode() == ISD::SELECT_CC)
    1055             :         Action = TLI.getOperationAction(Node->getOpcode(),
    1056             :                                         Node->getValueType(0));
    1057             :       else
    1058             :         Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
    1059             :     }
    1060             :     break;
    1061             :   }
    1062             :   case ISD::LOAD:
    1063             :   case ISD::STORE:
    1064             :     // FIXME: Model these properly.  LOAD and STORE are complicated, and
    1065             :     // STORE expects the unlegalized operand in some cases.
    1066             :     SimpleFinishLegalizing = false;
    1067             :     break;
    1068             :   case ISD::CALLSEQ_START:
    1069             :   case ISD::CALLSEQ_END:
    1070             :     // FIXME: This shouldn't be necessary.  These nodes have special properties
    1071             :     // dealing with the recursive nature of legalization.  Removing this
    1072             :     // special case should be done as part of making LegalizeDAG non-recursive.
    1073             :     SimpleFinishLegalizing = false;
    1074             :     break;
    1075      148961 :   case ISD::EXTRACT_ELEMENT:
    1076             :   case ISD::FLT_ROUNDS_:
    1077             :   case ISD::MERGE_VALUES:
    1078             :   case ISD::EH_RETURN:
    1079             :   case ISD::FRAME_TO_ARGS_OFFSET:
    1080             :   case ISD::EH_DWARF_CFA:
    1081             :   case ISD::EH_SJLJ_SETJMP:
    1082             :   case ISD::EH_SJLJ_LONGJMP:
    1083             :   case ISD::EH_SJLJ_SETUP_DISPATCH:
    1084             :     // These operations lie about being legal: when they claim to be legal,
    1085             :     // they should actually be expanded.
    1086      148961 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1087      148961 :     if (Action == TargetLowering::Legal)
    1088             :       Action = TargetLowering::Expand;
    1089             :     break;
    1090         207 :   case ISD::INIT_TRAMPOLINE:
    1091             :   case ISD::ADJUST_TRAMPOLINE:
    1092             :   case ISD::FRAMEADDR:
    1093             :   case ISD::RETURNADDR:
    1094             :   case ISD::ADDROFRETURNADDR:
    1095             :     // These operations lie about being legal: when they claim to be legal,
    1096             :     // they should actually be custom-lowered.
    1097         207 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1098         207 :     if (Action == TargetLowering::Legal)
    1099             :       Action = TargetLowering::Custom;
    1100             :     break;
    1101          25 :   case ISD::READCYCLECOUNTER:
    1102             :     // READCYCLECOUNTER returns an i64, even if type legalization might have
    1103             :     // expanded that to several smaller types.
    1104          25 :     Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
    1105             :     break;
    1106             :   case ISD::READ_REGISTER:
    1107             :   case ISD::WRITE_REGISTER:
    1108             :     // Named register is legal in the DAG, but blocked by register name
    1109             :     // selection if not implemented by target (to chose the correct register)
    1110             :     // They'll be converted to Copy(To/From)Reg.
    1111             :     Action = TargetLowering::Legal;
    1112             :     break;
    1113          26 :   case ISD::DEBUGTRAP:
    1114          26 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1115          26 :     if (Action == TargetLowering::Expand) {
    1116             :       // replace ISD::DEBUGTRAP with ISD::TRAP
    1117             :       SDValue NewVal;
    1118          33 :       NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
    1119             :                            Node->getOperand(0));
    1120          11 :       ReplaceNode(Node, NewVal.getNode());
    1121          11 :       LegalizeOp(NewVal.getNode());
    1122             :       return;
    1123          15 :     }
    1124             :     break;
    1125          34 :   case ISD::STRICT_FSQRT:
    1126             :   case ISD::STRICT_FMA:
    1127             :   case ISD::STRICT_FPOW:
    1128             :   case ISD::STRICT_FPOWI:
    1129             :   case ISD::STRICT_FSIN:
    1130             :   case ISD::STRICT_FCOS:
    1131             :   case ISD::STRICT_FEXP:
    1132             :   case ISD::STRICT_FEXP2:
    1133             :   case ISD::STRICT_FLOG:
    1134             :   case ISD::STRICT_FLOG10:
    1135             :   case ISD::STRICT_FLOG2:
    1136             :   case ISD::STRICT_FRINT:
    1137             :   case ISD::STRICT_FNEARBYINT:
    1138             :     // These pseudo-ops get legalized as if they were their non-strict
    1139             :     // equivalent.  For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
    1140             :     // is also legal, but if ISD::FSQRT requires expansion then so does
    1141             :     // ISD::STRICT_FSQRT.
    1142          34 :     Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(),
    1143             :                                      Node->getValueType(0));
    1144             :     break;
    1145    16321592 :   default:
    1146    16321592 :     if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
    1147             :       Action = TargetLowering::Legal;
    1148             :     } else {
    1149    13342237 :       Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1150             :     }
    1151             :     break;
    1152             :   }
    1153             : 
    1154             :   if (SimpleFinishLegalizing) {
    1155             :     SDNode *NewNode = Node;
    1156    16847124 :     switch (Node->getOpcode()) {
    1157             :     default: break;
    1158      270547 :     case ISD::SHL:
    1159             :     case ISD::SRL:
    1160             :     case ISD::SRA:
    1161             :     case ISD::ROTL:
    1162             :     case ISD::ROTR: {
    1163             :       // Legalizing shifts/rotates requires adjusting the shift amount
    1164             :       // to the appropriate width.
    1165      270547 :       SDValue Op0 = Node->getOperand(0);
    1166      270547 :       SDValue Op1 = Node->getOperand(1);
    1167      541094 :       if (!Op1.getValueType().isVector()) {
    1168      534244 :         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
    1169             :         // The getShiftAmountOperand() may create a new operand node or
    1170             :         // return the existing one. If new operand is created we need
    1171             :         // to update the parent node.
    1172             :         // Do not try to legalize SAO here! It will be automatically legalized
    1173             :         // in the next round.
    1174             :         if (SAO != Op1)
    1175       42189 :           NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
    1176      270547 :       }
    1177             :     }
    1178      270547 :     break;
    1179         429 :     case ISD::SRL_PARTS:
    1180             :     case ISD::SRA_PARTS:
    1181             :     case ISD::SHL_PARTS: {
    1182             :       // Legalizing shifts/rotates requires adjusting the shift amount
    1183             :       // to the appropriate width.
    1184         429 :       SDValue Op0 = Node->getOperand(0);
    1185         429 :       SDValue Op1 = Node->getOperand(1);
    1186         429 :       SDValue Op2 = Node->getOperand(2);
    1187         858 :       if (!Op2.getValueType().isVector()) {
    1188         858 :         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
    1189             :         // The getShiftAmountOperand() may create a new operand node or
    1190             :         // return the existing one. If new operand is created we need
    1191             :         // to update the parent node.
    1192             :         if (SAO != Op2)
    1193           0 :           NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
    1194             :       }
    1195             :       break;
    1196             :     }
    1197             :     }
    1198             : 
    1199    16847124 :     if (NewNode != Node) {
    1200         112 :       ReplaceNode(Node, NewNode);
    1201             :       Node = NewNode;
    1202             :     }
    1203    16847124 :     switch (Action) {
    1204             :     case TargetLowering::Legal:
    1205             :       DEBUG(dbgs() << "Legal node: nothing to do\n");
    1206             :       return;
    1207     1063675 :     case TargetLowering::Custom:
    1208             :       DEBUG(dbgs() << "Trying custom legalization\n");
    1209             :       // FIXME: The handling for custom lowering with multiple results is
    1210             :       // a complete mess.
    1211     2127350 :       if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
    1212     1014630 :         if (!(Res.getNode() != Node || Res.getResNo() != 0))
    1213             :           return;
    1214             : 
    1215      808991 :         if (Node->getNumValues() == 1) {
    1216             :           DEBUG(dbgs() << "Successfully custom legalized node\n");
    1217             :           // We can just directly replace this node with the lowered value.
    1218      671561 :           ReplaceNode(SDValue(Node, 0), Res);
    1219      671561 :           return;
    1220             :         }
    1221             : 
    1222             :         SmallVector<SDValue, 8> ResultVals;
    1223      687278 :         for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
    1224      274924 :           ResultVals.push_back(Res.getValue(i));
    1225             :         DEBUG(dbgs() << "Successfully custom legalized node\n");
    1226      137430 :         ReplaceNode(Node, ResultVals.data());
    1227             :         return;
    1228     1014630 :       }
    1229             :       DEBUG(dbgs() << "Could not custom legalize node\n");
    1230             :       LLVM_FALLTHROUGH;
    1231             :     case TargetLowering::Expand:
    1232      224738 :       if (ExpandNode(Node))
    1233             :         return;
    1234             :       LLVM_FALLTHROUGH;
    1235             :     case TargetLowering::LibCall:
    1236       36334 :       ConvertNodeToLibcall(Node);
    1237       36334 :       return;
    1238        6493 :     case TargetLowering::Promote:
    1239        6493 :       PromoteNode(Node);
    1240        6493 :       return;
    1241             :     }
    1242             :   }
    1243             : 
    1244     3548802 :   switch (Node->getOpcode()) {
    1245           0 :   default:
    1246             : #ifndef NDEBUG
    1247             :     dbgs() << "NODE: ";
    1248             :     Node->dump( &DAG);
    1249             :     dbgs() << "\n";
    1250             : #endif
    1251           0 :     llvm_unreachable("Do not know how to legalize this operator!");
    1252             : 
    1253             :   case ISD::CALLSEQ_START:
    1254             :   case ISD::CALLSEQ_END:
    1255             :     break;
    1256     1157534 :   case ISD::LOAD:
    1257     1157534 :     return LegalizeLoadOps(Node);
    1258     1503261 :   case ISD::STORE:
    1259     1503261 :     return LegalizeStoreOps(Node);
    1260             :   }
    1261             : }
    1262             : 
    1263        2610 : SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
    1264        2610 :   SDValue Vec = Op.getOperand(0);
    1265        2610 :   SDValue Idx = Op.getOperand(1);
    1266             :   SDLoc dl(Op);
    1267             : 
    1268             :   // Before we generate a new store to a temporary stack slot, see if there is
    1269             :   // already one that we can use. There often is because when we scalarize
    1270             :   // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
    1271             :   // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
    1272             :   // the vector. If all are expanded here, we don't want one store per vector
    1273             :   // element.
    1274             : 
    1275             :   // Caches for hasPredecessorHelper
    1276             :   SmallPtrSet<const SDNode *, 32> Visited;
    1277             :   SmallVector<const SDNode *, 16> Worklist;
    1278        2610 :   Worklist.push_back(Idx.getNode());
    1279        2610 :   SDValue StackPtr, Ch;
    1280             :   for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
    1281        5246 :        UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
    1282             :     SDNode *User = *UI;
    1283             :     if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
    1284        4276 :       if (ST->isIndexed() || ST->isTruncatingStore() ||
    1285             :           ST->getValue() != Vec)
    1286             :         continue;
    1287             : 
    1288             :       // Make sure that nothing else could have stored into the destination of
    1289             :       // this store.
    1290        4276 :       if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
    1291             :         continue;
    1292             : 
    1293             :       // If the index is dependent on the store we will introduce a cycle when
    1294             :       // creating the load (the load uses the index, and by replacing the chain
    1295             :       // we will make the index dependent on the load). Also, the store might be
    1296             :       // dependent on the extractelement and introduce a cycle when creating
    1297             :       // the load.
    1298        4277 :       if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
    1299        2137 :           ST->hasPredecessor(Op.getNode()))
    1300             :         continue;
    1301             : 
    1302        2136 :       StackPtr = ST->getBasePtr();
    1303             :       Ch = SDValue(ST, 0);
    1304             :       break;
    1305             :     }
    1306             :   }
    1307             : 
    1308        2610 :   EVT VecVT = Vec.getValueType();
    1309             : 
    1310        2610 :   if (!Ch.getNode()) {
    1311             :     // Store the value to a temporary stack slot, then LOAD the returned part.
    1312         474 :     StackPtr = DAG.CreateStackTemporary(VecVT);
    1313         948 :     Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
    1314         474 :                       MachinePointerInfo());
    1315             :   }
    1316             : 
    1317        2610 :   StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1318             : 
    1319             :   SDValue NewLoad;
    1320             : 
    1321        5220 :   if (Op.getValueType().isVector())
    1322           0 :     NewLoad =
    1323           0 :         DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
    1324             :   else
    1325        5220 :     NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
    1326             :                              MachinePointerInfo(),
    1327        5220 :                              VecVT.getVectorElementType());
    1328             : 
    1329             :   // Replace the chain going out of the store, by the one out of the load.
    1330        5220 :   DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
    1331             : 
    1332             :   // We introduced a cycle though, so update the loads operands, making sure
    1333             :   // to use the original store's chain as an incoming chain.
    1334             :   SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
    1335             :                                           NewLoad->op_end());
    1336        2610 :   NewLoadOperands[0] = Ch;
    1337             :   NewLoad =
    1338        5220 :       SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
    1339        5220 :   return NewLoad;
    1340             : }
    1341             : 
    1342           0 : SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
    1343             :   assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
    1344             : 
    1345           0 :   SDValue Vec  = Op.getOperand(0);
    1346           0 :   SDValue Part = Op.getOperand(1);
    1347           0 :   SDValue Idx  = Op.getOperand(2);
    1348             :   SDLoc dl(Op);
    1349             : 
    1350             :   // Store the value to a temporary stack slot, then LOAD the returned part.
    1351           0 :   EVT VecVT = Vec.getValueType();
    1352           0 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1353             :   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1354             :   MachinePointerInfo PtrInfo =
    1355           0 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
    1356             : 
    1357             :   // First store the whole vector.
    1358           0 :   SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
    1359             : 
    1360             :   // Then store the inserted part.
    1361           0 :   SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1362             : 
    1363             :   // Store the subvector.
    1364           0 :   Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
    1365             : 
    1366             :   // Finally, load the updated vector.
    1367           0 :   return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
    1368             : }
    1369             : 
    1370         155 : SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
    1371             :   // We can't handle this case efficiently.  Allocate a sufficiently
    1372             :   // aligned object on the stack, store each element into it, then load
    1373             :   // the result as a vector.
    1374             :   // Create the stack frame object.
    1375         155 :   EVT VT = Node->getValueType(0);
    1376         155 :   EVT EltVT = VT.getVectorElementType();
    1377             :   SDLoc dl(Node);
    1378         155 :   SDValue FIPtr = DAG.CreateStackTemporary(VT);
    1379             :   int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
    1380             :   MachinePointerInfo PtrInfo =
    1381         155 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
    1382             : 
    1383             :   // Emit a store of each element to the stack slot.
    1384             :   SmallVector<SDValue, 8> Stores;
    1385         155 :   unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
    1386             :   // Store (in the right endianness) the elements to memory.
    1387        1227 :   for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
    1388             :     // Ignore undef elements.
    1389         536 :     if (Node->getOperand(i).isUndef()) continue;
    1390             : 
    1391         399 :     unsigned Offset = TypeByteSize*i;
    1392             : 
    1393         798 :     SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
    1394         798 :     Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
    1395             : 
    1396             :     // If the destination vector element type is narrower than the source
    1397             :     // element type, only store the bits necessary.
    1398         399 :     if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
    1399         282 :       Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
    1400             :                                          Node->getOperand(i), Idx,
    1401         188 :                                          PtrInfo.getWithOffset(Offset), EltVT));
    1402             :     } else
    1403         915 :       Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
    1404         610 :                                     Idx, PtrInfo.getWithOffset(Offset)));
    1405             :   }
    1406             : 
    1407         155 :   SDValue StoreChain;
    1408         155 :   if (!Stores.empty())    // Not all undef elements?
    1409         310 :     StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
    1410             :   else
    1411           0 :     StoreChain = DAG.getEntryNode();
    1412             : 
    1413             :   // Result is a load from the stack slot.
    1414         465 :   return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
    1415             : }
    1416             : 
    1417             : /// Bitcast a floating-point value to an integer value. Only bitcast the part
    1418             : /// containing the sign bit if the target has no integer value capable of
    1419             : /// holding all bits of the floating-point value.
    1420          79 : void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
    1421             :                                              const SDLoc &DL,
    1422             :                                              SDValue Value) const {
    1423          79 :   EVT FloatVT = Value.getValueType();
    1424          79 :   unsigned NumBits = FloatVT.getSizeInBits();
    1425          79 :   State.FloatVT = FloatVT;
    1426          79 :   EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
    1427             :   // Convert to an integer of the same size.
    1428          79 :   if (TLI.isTypeLegal(IVT)) {
    1429         150 :     State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
    1430          75 :     State.SignMask = APInt::getSignMask(NumBits);
    1431          75 :     State.SignBit = NumBits - 1;
    1432          75 :     return;
    1433             :   }
    1434             : 
    1435           4 :   auto &DataLayout = DAG.getDataLayout();
    1436             :   // Store the float to memory, then load the sign part out as an integer.
    1437           8 :   MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
    1438             :   // First create a temporary that is aligned for both the load and store.
    1439           8 :   SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
    1440             :   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1441             :   // Then store the float to it.
    1442           4 :   State.FloatPtr = StackPtr;
    1443           4 :   MachineFunction &MF = DAG.getMachineFunction();
    1444           4 :   State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
    1445           8 :   State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
    1446           4 :                              State.FloatPointerInfo);
    1447             : 
    1448             :   SDValue IntPtr;
    1449           4 :   if (DataLayout.isBigEndian()) {
    1450             :     assert(FloatVT.isByteSized() && "Unsupported floating point type!");
    1451             :     // Load out a legal integer with the same sign bit as the float.
    1452             :     IntPtr = StackPtr;
    1453           1 :     State.IntPointerInfo = State.FloatPointerInfo;
    1454             :   } else {
    1455             :     // Advance the pointer so that the loaded byte will contain the sign bit.
    1456           3 :     unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
    1457           6 :     IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
    1458           6 :                       DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
    1459           3 :     State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
    1460           3 :                                                              ByteOffset);
    1461             :   }
    1462             : 
    1463           4 :   State.IntPtr = IntPtr;
    1464           8 :   State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
    1465           4 :                                   State.IntPointerInfo, MVT::i8);
    1466           8 :   State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
    1467           4 :   State.SignBit = 7;
    1468             : }
    1469             : 
    1470             : /// Replace the integer value produced by getSignAsIntValue() with a new value
    1471             : /// and cast the result back to a floating-point type.
    1472          37 : SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
    1473             :                                               const SDLoc &DL,
    1474             :                                               SDValue NewIntValue) const {
    1475          37 :   if (!State.Chain)
    1476          72 :     return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
    1477             : 
    1478             :   // Override the part containing the sign bit in the value stored on the stack.
    1479           1 :   SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
    1480           1 :                                     State.IntPointerInfo, MVT::i8);
    1481           1 :   return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
    1482           1 :                      State.FloatPointerInfo);
    1483             : }
    1484             : 
    1485          42 : SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
    1486             :   SDLoc DL(Node);
    1487          42 :   SDValue Mag = Node->getOperand(0);
    1488          42 :   SDValue Sign = Node->getOperand(1);
    1489             : 
    1490             :   // Get sign bit into an integer value.
    1491             :   FloatSignAsInt SignAsInt;
    1492          42 :   getSignAsIntValue(SignAsInt, DL, Sign);
    1493             : 
    1494          42 :   EVT IntVT = SignAsInt.IntValue.getValueType();
    1495          42 :   SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
    1496          42 :   SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
    1497          42 :                                 SignMask);
    1498             : 
    1499             :   // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
    1500          42 :   EVT FloatVT = Mag.getValueType();
    1501          42 :   if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
    1502             :       TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
    1503          10 :     SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
    1504          10 :     SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
    1505           5 :     SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
    1506           5 :                                 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
    1507           5 :     return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
    1508             :   }
    1509             : 
    1510             :   // Transform Mag value to integer, and clear the sign bit.
    1511             :   FloatSignAsInt MagAsInt;
    1512          37 :   getSignAsIntValue(MagAsInt, DL, Mag);
    1513          37 :   EVT MagVT = MagAsInt.IntValue.getValueType();
    1514         148 :   SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
    1515          37 :   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
    1516          37 :                                     ClearSignMask);
    1517             : 
    1518             :   // Get the signbit at the right position for MagAsInt.
    1519          37 :   int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
    1520          37 :   if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
    1521          19 :     if (ShiftAmount > 0) {
    1522          19 :       SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
    1523          38 :       SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
    1524           0 :     } else if (ShiftAmount < 0) {
    1525           0 :       SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
    1526           0 :       SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
    1527             :     }
    1528          38 :     SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
    1529          18 :   } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
    1530           0 :     SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
    1531           0 :     if (ShiftAmount > 0) {
    1532           0 :       SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
    1533           0 :       SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
    1534           0 :     } else if (ShiftAmount < 0) {
    1535           0 :       SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
    1536           0 :       SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
    1537             :     }
    1538             :   }
    1539             : 
    1540             :   // Store the part with the modified sign and convert back to float.
    1541          74 :   SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
    1542          37 :   return modifySignAsInt(MagAsInt, DL, CopiedSign);
    1543             : }
    1544             : 
    1545           2 : SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
    1546             :   SDLoc DL(Node);
    1547           2 :   SDValue Value = Node->getOperand(0);
    1548             : 
    1549             :   // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
    1550           2 :   EVT FloatVT = Value.getValueType();
    1551           2 :   if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
    1552           2 :     SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
    1553           4 :     return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
    1554             :   }
    1555             : 
    1556             :   // Transform value to integer, clear the sign bit and transform back.
    1557             :   FloatSignAsInt ValueAsInt;
    1558           0 :   getSignAsIntValue(ValueAsInt, DL, Value);
    1559           0 :   EVT IntVT = ValueAsInt.IntValue.getValueType();
    1560           0 :   SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
    1561           0 :   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
    1562           0 :                                     ClearSignMask);
    1563           0 :   return modifySignAsInt(ValueAsInt, DL, ClearedSign);
    1564             : }
    1565             : 
    1566         139 : void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
    1567             :                                            SmallVectorImpl<SDValue> &Results) {
    1568         139 :   unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
    1569             :   assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
    1570             :           " not tell us which reg is the stack pointer!");
    1571             :   SDLoc dl(Node);
    1572         139 :   EVT VT = Node->getValueType(0);
    1573             :   SDValue Tmp1 = SDValue(Node, 0);
    1574             :   SDValue Tmp2 = SDValue(Node, 1);
    1575         139 :   SDValue Tmp3 = Node->getOperand(2);
    1576         139 :   SDValue Chain = Tmp1.getOperand(0);
    1577             : 
    1578             :   // Chain the dynamic stack allocation so that it doesn't modify the stack
    1579             :   // pointer when other instructions are using the stack.
    1580         139 :   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
    1581             : 
    1582         139 :   SDValue Size  = Tmp2.getOperand(1);
    1583         139 :   SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
    1584         139 :   Chain = SP.getValue(1);
    1585         139 :   unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
    1586             :   unsigned StackAlign =
    1587         139 :       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
    1588         278 :   Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size);       // Value
    1589         139 :   if (Align > StackAlign)
    1590         102 :     Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
    1591         102 :                        DAG.getConstant(-(uint64_t)Align, dl, VT));
    1592         139 :   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
    1593             : 
    1594         417 :   Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
    1595         417 :                             DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
    1596             : 
    1597         139 :   Results.push_back(Tmp1);
    1598         139 :   Results.push_back(Tmp2);
    1599         139 : }
    1600             : 
    1601             : /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
    1602             : /// target.
    1603             : ///
    1604             : /// If the SETCC has been legalized using AND / OR, then the legalized node
    1605             : /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
    1606             : /// will be set to false.
    1607             : ///
    1608             : /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
    1609             : /// then the values of LHS and RHS will be swapped, CC will be set to the
    1610             : /// new condition, and NeedInvert will be set to false.
    1611             : ///
    1612             : /// If the SETCC has been legalized using the inverse condcode, then LHS and
    1613             : /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
    1614             : /// will be set to true. The caller must invert the result of the SETCC with
    1615             : /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
    1616             : /// of a true/false result.
    1617             : ///
    1618             : /// \returns true if the SetCC has been legalized, false if it hasn't.
    1619        2328 : bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
    1620             :                                                  SDValue &RHS, SDValue &CC,
    1621             :                                                  bool &NeedInvert,
    1622             :                                                  const SDLoc &dl) {
    1623             :   MVT OpVT = LHS.getSimpleValueType();
    1624             :   ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
    1625        2328 :   NeedInvert = false;
    1626             :   bool NeedSwap = false;
    1627        4656 :   switch (TLI.getCondCodeAction(CCCode, OpVT)) {
    1628           0 :   default: llvm_unreachable("Unknown condition code action!");
    1629             :   case TargetLowering::Legal:
    1630             :     // Nothing to do.
    1631             :     break;
    1632         325 :   case TargetLowering::Expand: {
    1633         325 :     ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
    1634         325 :     if (TLI.isCondCodeLegal(InvCC, OpVT)) {
    1635             :       std::swap(LHS, RHS);
    1636         166 :       CC = DAG.getCondCode(InvCC);
    1637             :       return true;
    1638             :     }
    1639             :     // Swapping operands didn't work. Try inverting the condition.
    1640         159 :     InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
    1641         159 :     if (!TLI.isCondCodeLegal(InvCC, OpVT)) {
    1642             :       // If inverting the condition is not enough, try swapping operands
    1643             :       // on top of it.
    1644         124 :       InvCC = ISD::getSetCCSwappedOperands(InvCC);
    1645             :       NeedSwap = true;
    1646             :     }
    1647         159 :     if (TLI.isCondCodeLegal(InvCC, OpVT)) {
    1648          48 :       CC = DAG.getCondCode(InvCC);
    1649          48 :       NeedInvert = true;
    1650          48 :       if (NeedSwap)
    1651             :         std::swap(LHS, RHS);
    1652             :       return true;
    1653             :     }
    1654             : 
    1655             :     ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
    1656             :     unsigned Opc = 0;
    1657         111 :     switch (CCCode) {
    1658           0 :     default: llvm_unreachable("Don't know how to expand this condition!");
    1659             :     case ISD::SETO:
    1660             :         assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
    1661             :             == TargetLowering::Legal
    1662             :             && "If SETO is expanded, SETOEQ must be legal!");
    1663             :         CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
    1664           9 :     case ISD::SETUO:
    1665             :         assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
    1666             :             == TargetLowering::Legal
    1667             :             && "If SETUO is expanded, SETUNE must be legal!");
    1668             :         CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;  break;
    1669             :     case ISD::SETOEQ:
    1670             :     case ISD::SETOGT:
    1671             :     case ISD::SETOGE:
    1672             :     case ISD::SETOLT:
    1673             :     case ISD::SETOLE:
    1674             :     case ISD::SETONE:
    1675             :     case ISD::SETUEQ:
    1676             :     case ISD::SETUNE:
    1677             :     case ISD::SETUGT:
    1678             :     case ISD::SETUGE:
    1679             :     case ISD::SETULT:
    1680             :     case ISD::SETULE:
    1681             :         // If we are floating point, assign and break, otherwise fall through.
    1682          96 :         if (!OpVT.isInteger()) {
    1683             :           // We can use the 4th bit to tell if we are the unordered
    1684             :           // or ordered version of the opcode.
    1685          96 :           CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
    1686          96 :           Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
    1687          96 :           CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
    1688             :           break;
    1689             :         }
    1690             :         // Fallthrough if we are unsigned integer.
    1691             :         LLVM_FALLTHROUGH;
    1692             :     case ISD::SETLE:
    1693             :     case ISD::SETGT:
    1694             :     case ISD::SETGE:
    1695             :     case ISD::SETLT:
    1696             :     case ISD::SETNE:
    1697             :     case ISD::SETEQ:
    1698             :       // If all combinations of inverting the condition and swapping operands
    1699             :       // didn't work then we have no means to expand the condition.
    1700           0 :       llvm_unreachable("Don't know how to expand this condition!");
    1701             :     }
    1702             : 
    1703         111 :     SDValue SetCC1, SetCC2;
    1704         111 :     if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
    1705             :       // If we aren't the ordered or unorder operation,
    1706             :       // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
    1707          96 :       SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
    1708          96 :       SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
    1709             :     } else {
    1710             :       // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
    1711          15 :       SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
    1712          15 :       SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
    1713             :     }
    1714         222 :     LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
    1715         111 :     RHS = SDValue();
    1716         111 :     CC  = SDValue();
    1717             :     return true;
    1718             :   }
    1719             :   }
    1720             :   return false;
    1721             : }
    1722             : 
    1723             : /// Emit a store/load combination to the stack.  This stores
    1724             : /// SrcOp to a stack slot of type SlotVT, truncating it if needed.  It then does
    1725             : /// a load from the stack slot to DestVT, extending it if needed.
    1726             : /// The resultant code need not be legal.
    1727         400 : SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
    1728             :                                                EVT DestVT, const SDLoc &dl) {
    1729             :   // Create the stack frame object.
    1730         800 :   unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
    1731         800 :       SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
    1732         400 :   SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
    1733             : 
    1734             :   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
    1735             :   int SPFI = StackPtrFI->getIndex();
    1736             :   MachinePointerInfo PtrInfo =
    1737         400 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
    1738             : 
    1739         400 :   unsigned SrcSize = SrcOp.getValueSizeInBits();
    1740         400 :   unsigned SlotSize = SlotVT.getSizeInBits();
    1741         400 :   unsigned DestSize = DestVT.getSizeInBits();
    1742         400 :   Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
    1743         800 :   unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
    1744             : 
    1745             :   // Emit a store to the stack slot.  Use a truncstore if the input value is
    1746             :   // later than DestVT.
    1747         400 :   SDValue Store;
    1748             : 
    1749         400 :   if (SrcSize > SlotSize)
    1750           0 :     Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
    1751           0 :                               SlotVT, SrcAlign);
    1752             :   else {
    1753             :     assert(SrcSize == SlotSize && "Invalid store");
    1754         400 :     Store =
    1755        1200 :         DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
    1756             :   }
    1757             : 
    1758             :   // Result is a load from the stack slot.
    1759         400 :   if (SlotSize == DestSize)
    1760         800 :     return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
    1761             : 
    1762             :   assert(SlotSize < DestSize && "Unknown extension!");
    1763           0 :   return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
    1764           0 :                         DestAlign);
    1765             : }
    1766             : 
    1767          18 : SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
    1768             :   SDLoc dl(Node);
    1769             :   // Create a vector sized/aligned stack slot, store the value to element #0,
    1770             :   // then load the whole vector back out.
    1771          36 :   SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
    1772             : 
    1773             :   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
    1774             :   int SPFI = StackPtrFI->getIndex();
    1775             : 
    1776          18 :   SDValue Ch = DAG.getTruncStore(
    1777          18 :       DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
    1778          18 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
    1779          54 :       Node->getValueType(0).getVectorElementType());
    1780          18 :   return DAG.getLoad(
    1781             :       Node->getValueType(0), dl, Ch, StackPtr,
    1782          54 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
    1783             : }
    1784             : 
    1785             : static bool
    1786          28 : ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
    1787             :                      const TargetLowering &TLI, SDValue &Res) {
    1788             :   unsigned NumElems = Node->getNumOperands();
    1789             :   SDLoc dl(Node);
    1790          28 :   EVT VT = Node->getValueType(0);
    1791             : 
    1792             :   // Try to group the scalars into pairs, shuffle the pairs together, then
    1793             :   // shuffle the pairs of pairs together, etc. until the vector has
    1794             :   // been built. This will work only if all of the necessary shuffle masks
    1795             :   // are legal.
    1796             : 
    1797             :   // We do this in two phases; first to check the legality of the shuffles,
    1798             :   // and next, assuming that all shuffles are legal, to create the new nodes.
    1799         140 :   for (int Phase = 0; Phase < 2; ++Phase) {
    1800          56 :     SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
    1801          56 :                                                               NewIntermedVals;
    1802         808 :     for (unsigned i = 0; i < NumElems; ++i) {
    1803         376 :       SDValue V = Node->getOperand(i);
    1804         376 :       if (V.isUndef())
    1805          16 :         continue;
    1806             : 
    1807             :       SDValue Vec;
    1808         360 :       if (Phase)
    1809         180 :         Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
    1810        1440 :       IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
    1811             :     }
    1812             : 
    1813         220 :     while (IntermedVals.size() > 2) {
    1814          82 :       NewIntermedVals.clear();
    1815         330 :       for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
    1816             :         // This vector and the next vector are shuffled together (simply to
    1817             :         // append the one to the other).
    1818         496 :         SmallVector<int, 16> ShuffleVec(NumElems, -1);
    1819             : 
    1820             :         SmallVector<int, 16> FinalIndices;
    1821         496 :         FinalIndices.reserve(IntermedVals[i].second.size() +
    1822         248 :                              IntermedVals[i+1].second.size());
    1823             : 
    1824             :         int k = 0;
    1825         596 :         for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
    1826             :              ++j, ++k) {
    1827         696 :           ShuffleVec[k] = j;
    1828         696 :           FinalIndices.push_back(IntermedVals[i].second[j]);
    1829             :         }
    1830         596 :         for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
    1831             :              ++j, ++k) {
    1832         696 :           ShuffleVec[k] = NumElems + j;
    1833         696 :           FinalIndices.push_back(IntermedVals[i+1].second[j]);
    1834             :         }
    1835             : 
    1836             :         SDValue Shuffle;
    1837         248 :         if (Phase)
    1838         124 :           Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
    1839             :                                          IntermedVals[i+1].first,
    1840         124 :                                          ShuffleVec);
    1841         248 :         else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
    1842             :           return false;
    1843         248 :         NewIntermedVals.push_back(
    1844         248 :             std::make_pair(Shuffle, std::move(FinalIndices)));
    1845             :       }
    1846             : 
    1847             :       // If we had an odd number of defined values, then append the last
    1848             :       // element to the array of new vectors.
    1849          82 :       if ((IntermedVals.size() & 1) != 0)
    1850           0 :         NewIntermedVals.push_back(IntermedVals.back());
    1851             : 
    1852          82 :       IntermedVals.swap(NewIntermedVals);
    1853             :     }
    1854             : 
    1855             :     assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
    1856             :            "Invalid number of intermediate vectors");
    1857          56 :     SDValue Vec1 = IntermedVals[0].first;
    1858          56 :     SDValue Vec2;
    1859          56 :     if (IntermedVals.size() > 1)
    1860          56 :       Vec2 = IntermedVals[1].first;
    1861           0 :     else if (Phase)
    1862           0 :       Vec2 = DAG.getUNDEF(VT);
    1863             : 
    1864         112 :     SmallVector<int, 16> ShuffleVec(NumElems, -1);
    1865         236 :     for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
    1866         540 :       ShuffleVec[IntermedVals[0].second[i]] = i;
    1867         236 :     for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
    1868         540 :       ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
    1869             : 
    1870          56 :     if (Phase)
    1871          28 :       Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
    1872          56 :     else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
    1873             :       return false;
    1874             :   }
    1875             : 
    1876             :   return true;
    1877             : }
    1878             : 
    1879             : /// Expand a BUILD_VECTOR node on targets that don't
    1880             : /// support the operation, but do support the resultant vector type.
    1881       20573 : SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
    1882             :   unsigned NumElems = Node->getNumOperands();
    1883             :   SDValue Value1, Value2;
    1884             :   SDLoc dl(Node);
    1885       20573 :   EVT VT = Node->getValueType(0);
    1886             :   EVT OpVT = Node->getOperand(0).getValueType();
    1887       20573 :   EVT EltVT = VT.getVectorElementType();
    1888             : 
    1889             :   // If the only non-undef value is the low element, turn this into a
    1890             :   // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
    1891             :   bool isOnlyLowElement = true;
    1892             :   bool MoreThanTwoValues = false;
    1893             :   bool isConstant = true;
    1894      516305 :   for (unsigned i = 0; i < NumElems; ++i) {
    1895      247866 :     SDValue V = Node->getOperand(i);
    1896      247866 :     if (V.isUndef())
    1897             :       continue;
    1898      233391 :     if (i > 0)
    1899             :       isOnlyLowElement = false;
    1900             :     if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
    1901             :       isConstant = false;
    1902             : 
    1903      233391 :     if (!Value1.getNode()) {
    1904             :       Value1 = V;
    1905      212818 :     } else if (!Value2.getNode()) {
    1906             :       if (V != Value1)
    1907             :         Value2 = V;
    1908             :     } else if (V != Value1 && V != Value2) {
    1909             :       MoreThanTwoValues = true;
    1910             :     }
    1911             :   }
    1912             : 
    1913       20573 :   if (!Value1.getNode())
    1914           0 :     return DAG.getUNDEF(VT);
    1915             : 
    1916       20573 :   if (isOnlyLowElement)
    1917          30 :     return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
    1918             : 
    1919             :   // If all elements are constants, create a load from the constant pool.
    1920       20558 :   if (isConstant) {
    1921             :     SmallVector<Constant*, 16> CV;
    1922      497063 :     for (unsigned i = 0, e = NumElems; i != e; ++i) {
    1923             :       if (ConstantFPSDNode *V =
    1924             :           dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
    1925        5725 :         CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
    1926             :       } else if (ConstantSDNode *V =
    1927             :                  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
    1928           0 :         if (OpVT==EltVT)
    1929      215986 :           CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
    1930             :         else {
    1931             :           // If OpVT and EltVT don't match, EltVT is not legal and the
    1932             :           // element values have been promoted/truncated earlier.  Undo this;
    1933             :           // we don't want a v16i8 to become a v16i32 for example.
    1934             :           const ConstantInt *CI = V->getConstantIntValue();
    1935        3198 :           CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
    1936             :                                         CI->getZExtValue()));
    1937             :         }
    1938             :       } else {
    1939             :         assert(Node->getOperand(i).isUndef());
    1940       14227 :         Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
    1941       14227 :         CV.push_back(UndefValue::get(OpNTy));
    1942             :       }
    1943             :     }
    1944       18791 :     Constant *CP = ConstantVector::get(CV);
    1945             :     SDValue CPIdx =
    1946       37582 :         DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
    1947             :     unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
    1948       18791 :     return DAG.getLoad(
    1949       18791 :         VT, dl, DAG.getEntryNode(), CPIdx,
    1950             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
    1951       37582 :         Alignment);
    1952             :   }
    1953             : 
    1954        1767 :   SmallSet<SDValue, 16> DefinedValues;
    1955       19163 :   for (unsigned i = 0; i < NumElems; ++i) {
    1956        8698 :     if (Node->getOperand(i).isUndef())
    1957         231 :       continue;
    1958        8467 :     DefinedValues.insert(Node->getOperand(i));
    1959             :   }
    1960             : 
    1961        3534 :   if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
    1962        1703 :     if (!MoreThanTwoValues) {
    1963        3350 :       SmallVector<int, 8> ShuffleVec(NumElems, -1);
    1964       18315 :       for (unsigned i = 0; i < NumElems; ++i) {
    1965        8320 :         SDValue V = Node->getOperand(i);
    1966        8320 :         if (V.isUndef())
    1967             :           continue;
    1968        9218 :         ShuffleVec[i] = V == Value1 ? 0 : NumElems;
    1969             :       }
    1970        3350 :       if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
    1971             :         // Get the splatted value into the low element of a vector register.
    1972        3168 :         SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
    1973        1584 :         SDValue Vec2;
    1974        1584 :         if (Value2.getNode())
    1975        2050 :           Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
    1976             :         else
    1977         559 :           Vec2 = DAG.getUNDEF(VT);
    1978             : 
    1979             :         // Return shuffle(LowValVec, undef, <0,0,0,0>)
    1980        3168 :         return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
    1981             :       }
    1982             :     } else {
    1983          28 :       SDValue Res;
    1984          28 :       if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
    1985          28 :         return Res;
    1986             :     }
    1987             :   }
    1988             : 
    1989             :   // Otherwise, we can't handle this case efficiently.
    1990         155 :   return ExpandVectorBuildThroughStack(Node);
    1991             : }
    1992             : 
    1993             : // Expand a node into a call to a libcall.  If the result value
    1994             : // does not fit into a register, return the lo part and set the hi part to the
    1995             : // by-reg argument.  If it does fit into a single register, return the result
    1996             : // and leave the Hi part unset.
    1997        2356 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
    1998             :                                             bool isSigned) {
    1999             :   TargetLowering::ArgListTy Args;
    2000             :   TargetLowering::ArgListEntry Entry;
    2001        5694 :   for (const SDValue &Op : Node->op_values()) {
    2002        3338 :     EVT ArgVT = Op.getValueType();
    2003        3338 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2004        3338 :     Entry.Node = Op;
    2005        3338 :     Entry.Ty = ArgTy;
    2006        3338 :     Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
    2007        3338 :     Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
    2008        3338 :     Args.push_back(Entry);
    2009             :   }
    2010        4712 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2011        2356 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2012             : 
    2013        2356 :   EVT RetVT = Node->getValueType(0);
    2014        2356 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2015             : 
    2016             :   // By default, the input chain to this libcall is the entry node of the
    2017             :   // function. If the libcall is going to be emitted as a tail call then
    2018             :   // TLI.isUsedByReturnOnly will change it to the right chain if the return
    2019             :   // node which is being folded has a non-entry input chain.
    2020        2356 :   SDValue InChain = DAG.getEntryNode();
    2021             : 
    2022             :   // isTailCall may be true since the callee does not reference caller stack
    2023             :   // frame. Check if it's in the right position and that the return types match.
    2024        2356 :   SDValue TCChain = InChain;
    2025             :   const Function &F = DAG.getMachineFunction().getFunction();
    2026             :   bool isTailCall =
    2027        2702 :       TLI.isInTailCallPosition(DAG, Node, TCChain) &&
    2028           6 :       (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
    2029             :   if (isTailCall)
    2030         340 :     InChain = TCChain;
    2031             : 
    2032        4712 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2033        2356 :   bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
    2034        2356 :   CLI.setDebugLoc(SDLoc(Node))
    2035             :       .setChain(InChain)
    2036        2356 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2037        2356 :                     std::move(Args))
    2038             :       .setTailCall(isTailCall)
    2039             :       .setSExtResult(signExtend)
    2040        2356 :       .setZExtResult(!signExtend)
    2041             :       .setIsPostTypeLegalization(true);
    2042             : 
    2043        2356 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2044             : 
    2045        2356 :   if (!CallInfo.second.getNode()) {
    2046             :     DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump());
    2047             :     // It's a tailcall, return the chain (which is the DAG root).
    2048         308 :     return DAG.getRoot();
    2049             :   }
    2050             : 
    2051             :   DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump());
    2052        2048 :   return CallInfo.first;
    2053             : }
    2054             : 
    2055             : /// Generate a libcall taking the given operands as arguments
    2056             : /// and returning a result of type RetVT.
    2057           5 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
    2058             :                                             const SDValue *Ops, unsigned NumOps,
    2059             :                                             bool isSigned, const SDLoc &dl) {
    2060             :   TargetLowering::ArgListTy Args;
    2061           5 :   Args.reserve(NumOps);
    2062             : 
    2063             :   TargetLowering::ArgListEntry Entry;
    2064          45 :   for (unsigned i = 0; i != NumOps; ++i) {
    2065          20 :     Entry.Node = Ops[i];
    2066          20 :     Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
    2067          20 :     Entry.IsSExt = isSigned;
    2068          20 :     Entry.IsZExt = !isSigned;
    2069          20 :     Args.push_back(Entry);
    2070             :   }
    2071          10 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2072           5 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2073             : 
    2074           5 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2075             : 
    2076          10 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2077             :   CLI.setDebugLoc(dl)
    2078           5 :       .setChain(DAG.getEntryNode())
    2079           5 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2080           5 :                     std::move(Args))
    2081             :       .setSExtResult(isSigned)
    2082           5 :       .setZExtResult(!isSigned)
    2083             :       .setIsPostTypeLegalization(true);
    2084             : 
    2085           5 :   std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2086             : 
    2087          10 :   return CallInfo.first;
    2088             : }
    2089             : 
    2090             : // Expand a node into a call to a libcall. Similar to
    2091             : // ExpandLibCall except that the first operand is the in-chain.
    2092             : std::pair<SDValue, SDValue>
    2093         309 : SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
    2094             :                                          SDNode *Node,
    2095             :                                          bool isSigned) {
    2096         309 :   SDValue InChain = Node->getOperand(0);
    2097             : 
    2098             :   TargetLowering::ArgListTy Args;
    2099             :   TargetLowering::ArgListEntry Entry;
    2100        1579 :   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
    2101         635 :     EVT ArgVT = Node->getOperand(i).getValueType();
    2102         635 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2103         635 :     Entry.Node = Node->getOperand(i);
    2104         635 :     Entry.Ty = ArgTy;
    2105         635 :     Entry.IsSExt = isSigned;
    2106         635 :     Entry.IsZExt = !isSigned;
    2107         635 :     Args.push_back(Entry);
    2108             :   }
    2109         618 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2110         309 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2111             : 
    2112         309 :   Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
    2113             : 
    2114         618 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2115         309 :   CLI.setDebugLoc(SDLoc(Node))
    2116             :       .setChain(InChain)
    2117         309 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2118         309 :                     std::move(Args))
    2119             :       .setSExtResult(isSigned)
    2120         309 :       .setZExtResult(!isSigned);
    2121             : 
    2122         309 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2123             : 
    2124         309 :   return CallInfo;
    2125             : }
    2126             : 
    2127        1437 : SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
    2128             :                                               RTLIB::Libcall Call_F32,
    2129             :                                               RTLIB::Libcall Call_F64,
    2130             :                                               RTLIB::Libcall Call_F80,
    2131             :                                               RTLIB::Libcall Call_F128,
    2132             :                                               RTLIB::Libcall Call_PPCF128) {
    2133             :   if (Node->isStrictFPOpcode())
    2134          22 :     Node = DAG.mutateStrictFPToFP(Node);
    2135             : 
    2136             :   RTLIB::Libcall LC;
    2137        1437 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2138           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2139             :   case MVT::f32: LC = Call_F32; break;
    2140         485 :   case MVT::f64: LC = Call_F64; break;
    2141          43 :   case MVT::f80: LC = Call_F80; break;
    2142          46 :   case MVT::f128: LC = Call_F128; break;
    2143           0 :   case MVT::ppcf128: LC = Call_PPCF128; break;
    2144             :   }
    2145        1437 :   return ExpandLibCall(LC, Node, false);
    2146             : }
    2147             : 
    2148         116 : SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
    2149             :                                                RTLIB::Libcall Call_I8,
    2150             :                                                RTLIB::Libcall Call_I16,
    2151             :                                                RTLIB::Libcall Call_I32,
    2152             :                                                RTLIB::Libcall Call_I64,
    2153             :                                                RTLIB::Libcall Call_I128) {
    2154             :   RTLIB::Libcall LC;
    2155         116 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2156           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2157             :   case MVT::i8:   LC = Call_I8; break;
    2158          14 :   case MVT::i16:  LC = Call_I16; break;
    2159         102 :   case MVT::i32:  LC = Call_I32; break;
    2160           0 :   case MVT::i64:  LC = Call_I64; break;
    2161           0 :   case MVT::i128: LC = Call_I128; break;
    2162             :   }
    2163         116 :   return ExpandLibCall(LC, Node, isSigned);
    2164             : }
    2165             : 
    2166             : /// Issue libcalls to __{u}divmod to compute div / rem pairs.
    2167             : void
    2168          14 : SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
    2169             :                                           SmallVectorImpl<SDValue> &Results) {
    2170             :   unsigned Opcode = Node->getOpcode();
    2171          14 :   bool isSigned = Opcode == ISD::SDIVREM;
    2172             : 
    2173             :   RTLIB::Libcall LC;
    2174          14 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2175           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2176           0 :   case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
    2177           0 :   case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
    2178          14 :   case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
    2179           0 :   case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
    2180           0 :   case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
    2181             :   }
    2182             : 
    2183             :   // The input chain to this libcall is the entry node of the function.
    2184             :   // Legalizing the call will automatically add the previous call to the
    2185             :   // dependence.
    2186          14 :   SDValue InChain = DAG.getEntryNode();
    2187             : 
    2188          14 :   EVT RetVT = Node->getValueType(0);
    2189          14 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2190             : 
    2191             :   TargetLowering::ArgListTy Args;
    2192             :   TargetLowering::ArgListEntry Entry;
    2193          42 :   for (const SDValue &Op : Node->op_values()) {
    2194          28 :     EVT ArgVT = Op.getValueType();
    2195          28 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2196          28 :     Entry.Node = Op;
    2197          28 :     Entry.Ty = ArgTy;
    2198          28 :     Entry.IsSExt = isSigned;
    2199          28 :     Entry.IsZExt = !isSigned;
    2200          28 :     Args.push_back(Entry);
    2201             :   }
    2202             : 
    2203             :   // Also pass the return address of the remainder.
    2204          14 :   SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
    2205          14 :   Entry.Node = FIPtr;
    2206          14 :   Entry.Ty = RetTy->getPointerTo();
    2207          14 :   Entry.IsSExt = isSigned;
    2208          14 :   Entry.IsZExt = !isSigned;
    2209          14 :   Args.push_back(Entry);
    2210             : 
    2211          28 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2212          14 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2213             : 
    2214             :   SDLoc dl(Node);
    2215          28 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2216             :   CLI.setDebugLoc(dl)
    2217             :       .setChain(InChain)
    2218          14 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2219          14 :                     std::move(Args))
    2220             :       .setSExtResult(isSigned)
    2221             :       .setZExtResult(!isSigned);
    2222             : 
    2223          14 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2224             : 
    2225             :   // Remainder is loaded back from the stack frame.
    2226             :   SDValue Rem =
    2227          28 :       DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
    2228          14 :   Results.push_back(CallInfo.first);
    2229          14 :   Results.push_back(Rem);
    2230          14 : }
    2231             : 
    2232             : /// Return true if sincos libcall is available.
    2233             : static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
    2234             :   RTLIB::Libcall LC;
    2235         267 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2236           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2237             :   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
    2238          92 :   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
    2239          19 :   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
    2240          14 :   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
    2241           0 :   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
    2242             :   }
    2243             :   return TLI.getLibcallName(LC) != nullptr;
    2244             : }
    2245             : 
    2246             : /// Only issue sincos libcall if both sin and cos are needed.
    2247             : static bool useSinCos(SDNode *Node) {
    2248             :   unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
    2249         139 :     ? ISD::FCOS : ISD::FSIN;
    2250             : 
    2251         139 :   SDValue Op0 = Node->getOperand(0);
    2252             :   for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
    2253         200 :        UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
    2254             :     SDNode *User = *UI;
    2255         139 :     if (User == Node)
    2256             :       continue;
    2257             :     // The other user might have been turned into sincos already.
    2258          78 :     if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
    2259             :       return true;
    2260             :   }
    2261             :   return false;
    2262             : }
    2263             : 
    2264             : /// Issue libcalls to sincos to compute sin / cos pairs.
    2265             : void
    2266          30 : SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
    2267             :                                           SmallVectorImpl<SDValue> &Results) {
    2268             :   RTLIB::Libcall LC;
    2269          30 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2270           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2271             :   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
    2272          11 :   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
    2273           5 :   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
    2274           3 :   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
    2275           0 :   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
    2276             :   }
    2277             : 
    2278             :   // The input chain to this libcall is the entry node of the function.
    2279             :   // Legalizing the call will automatically add the previous call to the
    2280             :   // dependence.
    2281          30 :   SDValue InChain = DAG.getEntryNode();
    2282             : 
    2283          30 :   EVT RetVT = Node->getValueType(0);
    2284          30 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2285             : 
    2286             :   TargetLowering::ArgListTy Args;
    2287             :   TargetLowering::ArgListEntry Entry;
    2288             : 
    2289             :   // Pass the argument.
    2290          30 :   Entry.Node = Node->getOperand(0);
    2291          30 :   Entry.Ty = RetTy;
    2292             :   Entry.IsSExt = false;
    2293             :   Entry.IsZExt = false;
    2294          30 :   Args.push_back(Entry);
    2295             : 
    2296             :   // Pass the return address of sin.
    2297          30 :   SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
    2298          30 :   Entry.Node = SinPtr;
    2299          30 :   Entry.Ty = RetTy->getPointerTo();
    2300          30 :   Entry.IsSExt = false;
    2301          30 :   Entry.IsZExt = false;
    2302          30 :   Args.push_back(Entry);
    2303             : 
    2304             :   // Also pass the return address of the cos.
    2305          30 :   SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
    2306          30 :   Entry.Node = CosPtr;
    2307          30 :   Entry.Ty = RetTy->getPointerTo();
    2308          30 :   Entry.IsSExt = false;
    2309          30 :   Entry.IsZExt = false;
    2310          30 :   Args.push_back(Entry);
    2311             : 
    2312          60 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2313          30 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2314             : 
    2315             :   SDLoc dl(Node);
    2316          60 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2317          30 :   CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
    2318          60 :       TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
    2319          30 :       std::move(Args));
    2320             : 
    2321          30 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2322             : 
    2323          60 :   Results.push_back(
    2324          90 :       DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
    2325          30 :   Results.push_back(
    2326          90 :       DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
    2327          30 : }
    2328             : 
    2329             : /// This function is responsible for legalizing a
    2330             : /// INT_TO_FP operation of the specified operand when the target requests that
    2331             : /// we expand it.  At this point, we know that the result and operand types are
    2332             : /// legal for the target.
    2333         137 : SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
    2334             :                                                    EVT DestVT,
    2335             :                                                    const SDLoc &dl) {
    2336             :   // TODO: Should any fast-math-flags be set for the created nodes?
    2337             :   DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
    2338          51 :   if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
    2339             :     DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
    2340             :                     "expansion\n");
    2341             : 
    2342             :     // Get the stack frame index of a 8 byte buffer.
    2343         100 :     SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
    2344             : 
    2345             :     // word offset constant for Hi/Lo address computation
    2346          50 :     SDValue WordOff = DAG.getConstant(sizeof(int), dl,
    2347          50 :                                       StackSlot.getValueType());
    2348             :     // set up Hi and Lo (into buffer) address based on endian
    2349          50 :     SDValue Hi = StackSlot;
    2350          50 :     SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
    2351          50 :                              StackSlot, WordOff);
    2352         100 :     if (DAG.getDataLayout().isLittleEndian())
    2353             :       std::swap(Hi, Lo);
    2354             : 
    2355             :     // if signed map to unsigned space
    2356          50 :     SDValue Op0Mapped;
    2357          50 :     if (isSigned) {
    2358             :       // constant used to invert sign bit (signed to unsigned mapping)
    2359           8 :       SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
    2360           8 :       Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
    2361             :     } else {
    2362          46 :       Op0Mapped = Op0;
    2363             :     }
    2364             :     // store the lo of the constructed double - based on integer input
    2365          50 :     SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
    2366          50 :                                   MachinePointerInfo());
    2367             :     // initial hi portion of constructed double
    2368         100 :     SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
    2369             :     // store the hi of the constructed double - biased exponent
    2370             :     SDValue Store2 =
    2371         100 :         DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
    2372             :     // load the constructed double
    2373             :     SDValue Load =
    2374         100 :         DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
    2375             :     // FP constant to bias correct the final result
    2376          50 :     SDValue Bias = DAG.getConstantFP(isSigned ?
    2377             :                                      BitsToDouble(0x4330000080000000ULL) :
    2378             :                                      BitsToDouble(0x4330000000000000ULL),
    2379          50 :                                      dl, MVT::f64);
    2380             :     // subtract the bias
    2381         100 :     SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
    2382             :     // final result
    2383             :     SDValue Result;
    2384             :     // handle final rounding
    2385             :     if (DestVT == MVT::f64) {
    2386             :       // do nothing
    2387          10 :       Result = Sub;
    2388          40 :     } else if (DestVT.bitsLT(MVT::f64)) {
    2389          72 :       Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
    2390          72 :                            DAG.getIntPtrConstant(0, dl));
    2391           4 :     } else if (DestVT.bitsGT(MVT::f64)) {
    2392           8 :       Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
    2393             :     }
    2394          50 :     return Result;
    2395             :   }
    2396             :   assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
    2397             :   // Code below here assumes !isSigned without checking again.
    2398             : 
    2399             :   // Implementation of unsigned i64 to f64 following the algorithm in
    2400             :   // __floatundidf in compiler_rt. This implementation has the advantage
    2401             :   // of performing rounding correctly, both in the default rounding mode
    2402             :   // and in all alternate rounding modes.
    2403             :   // TODO: Generalize this for use with other types.
    2404             :   if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
    2405             :     DEBUG(dbgs() << "Converting unsigned i64 to f64\n");
    2406             :     SDValue TwoP52 =
    2407           8 :       DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
    2408             :     SDValue TwoP84PlusTwoP52 =
    2409           4 :       DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
    2410           4 :                         MVT::f64);
    2411             :     SDValue TwoP84 =
    2412           8 :       DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
    2413             : 
    2414           8 :     SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
    2415           4 :     SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
    2416           8 :                              DAG.getConstant(32, dl, MVT::i64));
    2417           8 :     SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
    2418           8 :     SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
    2419           8 :     SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
    2420           8 :     SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
    2421           4 :     SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
    2422           4 :                                 TwoP84PlusTwoP52);
    2423           8 :     return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
    2424             :   }
    2425             : 
    2426             :   // TODO: Generalize this for use with other types.
    2427             :   if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
    2428             :     DEBUG(dbgs() << "Converting unsigned i64 to f32\n");
    2429             :     // For unsigned conversions, convert them to signed conversions using the
    2430             :     // algorithm from the x86_64 __floatundidf in compiler_rt.
    2431          81 :     if (!isSigned) {
    2432         162 :       SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
    2433             : 
    2434          81 :       SDValue ShiftConst = DAG.getConstant(
    2435         162 :           1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
    2436         162 :       SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
    2437         162 :       SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
    2438         162 :       SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
    2439         162 :       SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
    2440             : 
    2441         162 :       SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
    2442         162 :       SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
    2443             : 
    2444             :       // TODO: This really should be implemented using a branch rather than a
    2445             :       // select.  We happen to get lucky and machinesink does the right
    2446             :       // thing most of the time.  This would be a good candidate for a
    2447             :       //pseudo-op, or, even better, for whole-function isel.
    2448          81 :       SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
    2449         162 :         Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
    2450         162 :       return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
    2451             :     }
    2452             : 
    2453             :     // Otherwise, implement the fully general conversion.
    2454             : 
    2455           0 :     SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
    2456           0 :          DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
    2457           0 :     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
    2458           0 :          DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
    2459           0 :     SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
    2460           0 :          DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
    2461           0 :     SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
    2462             :                               DAG.getConstant(UINT64_C(0), dl, MVT::i64),
    2463           0 :                               ISD::SETNE);
    2464           0 :     SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
    2465           0 :     SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
    2466             :                               DAG.getConstant(UINT64_C(0x0020000000000000), dl,
    2467             :                                               MVT::i64),
    2468           0 :                               ISD::SETUGE);
    2469           0 :     SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
    2470           0 :     EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
    2471             : 
    2472           0 :     SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
    2473           0 :                              DAG.getConstant(32, dl, SHVT));
    2474           0 :     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
    2475           0 :     SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
    2476             :     SDValue TwoP32 =
    2477           0 :       DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
    2478           0 :                         MVT::f64);
    2479           0 :     SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
    2480           0 :     SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
    2481           0 :     SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
    2482           0 :     SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
    2483           0 :     return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
    2484           0 :                        DAG.getIntPtrConstant(0, dl));
    2485             :   }
    2486             : 
    2487           4 :   SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
    2488             : 
    2489           2 :   SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
    2490             :                                  Op0,
    2491             :                                  DAG.getConstant(0, dl, Op0.getValueType()),
    2492           4 :                                  ISD::SETLT);
    2493           2 :   SDValue Zero = DAG.getIntPtrConstant(0, dl),
    2494           2 :           Four = DAG.getIntPtrConstant(4, dl);
    2495           2 :   SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
    2496           2 :                                     SignSet, Four, Zero);
    2497             : 
    2498             :   // If the sign bit of the integer is set, the large number will be treated
    2499             :   // as a negative number.  To counteract this, the dynamic code adds an
    2500             :   // offset depending on the data type.
    2501             :   uint64_t FF;
    2502           2 :   switch (Op0.getSimpleValueType().SimpleTy) {
    2503           0 :   default: llvm_unreachable("Unsupported integer type!");
    2504             :   case MVT::i8 : FF = 0x43800000ULL; break;  // 2^8  (as a float)
    2505           0 :   case MVT::i16: FF = 0x47800000ULL; break;  // 2^16 (as a float)
    2506           1 :   case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
    2507           1 :   case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
    2508             :   }
    2509           4 :   if (DAG.getDataLayout().isLittleEndian())
    2510           0 :     FF <<= 32;
    2511           2 :   Constant *FudgeFactor = ConstantInt::get(
    2512           4 :                                        Type::getInt64Ty(*DAG.getContext()), FF);
    2513             : 
    2514             :   SDValue CPIdx =
    2515           4 :       DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
    2516           2 :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
    2517           4 :   CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
    2518           4 :   Alignment = std::min(Alignment, 4u);
    2519           2 :   SDValue FudgeInReg;
    2520             :   if (DestVT == MVT::f32)
    2521           2 :     FudgeInReg = DAG.getLoad(
    2522           1 :         MVT::f32, dl, DAG.getEntryNode(), CPIdx,
    2523             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
    2524           2 :         Alignment);
    2525             :   else {
    2526           1 :     SDValue Load = DAG.getExtLoad(
    2527           1 :         ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
    2528             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
    2529           2 :         Alignment);
    2530           2 :     HandleSDNode Handle(Load);
    2531           1 :     LegalizeOp(Load.getNode());
    2532           1 :     FudgeInReg = Handle.getValue();
    2533             :   }
    2534             : 
    2535           4 :   return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
    2536             : }
    2537             : 
    2538             : /// This function is responsible for legalizing a
    2539             : /// *INT_TO_FP operation of the specified operand when the target requests that
    2540             : /// we promote it.  At this point, we know that the result and operand types are
    2541             : /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
    2542             : /// operation that takes a larger input.
    2543          65 : SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
    2544             :                                                     bool isSigned,
    2545             :                                                     const SDLoc &dl) {
    2546             :   // First step, figure out the appropriate *INT_TO_FP operation to use.
    2547             :   EVT NewInTy = LegalOp.getValueType();
    2548             : 
    2549             :   unsigned OpToUse = 0;
    2550             : 
    2551             :   // Scan for the appropriate larger type to use.
    2552             :   while (true) {
    2553          71 :     NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
    2554             :     assert(NewInTy.isInteger() && "Ran out of possibilities!");
    2555             : 
    2556             :     // If the target supports SINT_TO_FP of this type, use it.
    2557          71 :     if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
    2558             :       OpToUse = ISD::SINT_TO_FP;
    2559             :       break;
    2560             :     }
    2561           6 :     if (isSigned) continue;
    2562             : 
    2563             :     // If the target supports UINT_TO_FP of this type, use it.
    2564             :     if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
    2565             :       OpToUse = ISD::UINT_TO_FP;
    2566             :       break;
    2567             :     }
    2568             : 
    2569             :     // Otherwise, try a larger type.
    2570             :   }
    2571             : 
    2572             :   // Okay, we found the operation and type to use.  Zero extend our input to the
    2573             :   // desired type then run the operation on it.
    2574          65 :   return DAG.getNode(OpToUse, dl, DestVT,
    2575             :                      DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
    2576          65 :                                  dl, NewInTy, LegalOp));
    2577             : }
    2578             : 
    2579             : /// This function is responsible for legalizing a
    2580             : /// FP_TO_*INT operation of the specified operand when the target requests that
    2581             : /// we promote it.  At this point, we know that the result and operand types are
    2582             : /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
    2583             : /// operation that returns a larger result.
    2584         108 : SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
    2585             :                                                     bool isSigned,
    2586             :                                                     const SDLoc &dl) {
    2587             :   // First step, figure out the appropriate FP_TO*INT operation to use.
    2588         108 :   EVT NewOutTy = DestVT;
    2589             : 
    2590             :   unsigned OpToUse = 0;
    2591             : 
    2592             :   // Scan for the appropriate larger type to use.
    2593             :   while (true) {
    2594         128 :     NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
    2595             :     assert(NewOutTy.isInteger() && "Ran out of possibilities!");
    2596             : 
    2597             :     // A larger signed type can hold all unsigned values of the requested type,
    2598             :     // so using FP_TO_SINT is valid
    2599         128 :     if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
    2600             :       OpToUse = ISD::FP_TO_SINT;
    2601             :       break;
    2602             :     }
    2603             : 
    2604             :     // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
    2605          20 :     if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
    2606             :       OpToUse = ISD::FP_TO_UINT;
    2607             :       break;
    2608             :     }
    2609             : 
    2610             :     // Otherwise, try a larger type.
    2611             :   }
    2612             : 
    2613             :   // Okay, we found the operation and type to use.
    2614         216 :   SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
    2615             : 
    2616             :   // Truncate the result of the extended FP_TO_*INT operation to the desired
    2617             :   // size.
    2618         216 :   return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
    2619             : }
    2620             : 
    2621             : /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
    2622          92 : SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
    2623          92 :   EVT VT = Op.getValueType();
    2624         184 :   EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2625             :   unsigned Sz = VT.getScalarSizeInBits();
    2626             : 
    2627          92 :   SDValue Tmp, Tmp2, Tmp3;
    2628             : 
    2629             :   // If we can, perform BSWAP first and then the mask+swap the i4, then i2
    2630             :   // and finally the i1 pairs.
    2631             :   // TODO: We can easily support i4/i2 legal types if any target ever does.
    2632          92 :   if (Sz >= 8 && isPowerOf2_32(Sz)) {
    2633             :     // Create the masks - repeating the pattern every byte.
    2634             :     APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
    2635             :     APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
    2636         756 :     for (unsigned J = 0; J != Sz; J += 8) {
    2637        1328 :       MaskHi4 = MaskHi4 | (0xF0ull << J);
    2638        1328 :       MaskLo4 = MaskLo4 | (0x0Full << J);
    2639        1328 :       MaskHi2 = MaskHi2 | (0xCCull << J);
    2640        1328 :       MaskLo2 = MaskLo2 | (0x33ull << J);
    2641        1328 :       MaskHi1 = MaskHi1 | (0xAAull << J);
    2642        1328 :       MaskLo1 = MaskLo1 | (0x55ull << J);
    2643             :     }
    2644             : 
    2645             :     // BSWAP if the type is wider than a single byte.
    2646         144 :     Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
    2647             : 
    2648             :     // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
    2649         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
    2650         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
    2651         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
    2652         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
    2653         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2654             : 
    2655             :     // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
    2656         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
    2657         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
    2658         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
    2659         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
    2660         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2661             : 
    2662             :     // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
    2663         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
    2664         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
    2665         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
    2666         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
    2667         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2668          92 :     return Tmp;
    2669             :   }
    2670             : 
    2671           0 :   Tmp = DAG.getConstant(0, dl, VT);
    2672           0 :   for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
    2673           0 :     if (I < J)
    2674           0 :       Tmp2 =
    2675           0 :           DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
    2676             :     else
    2677           0 :       Tmp2 =
    2678           0 :           DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
    2679             : 
    2680             :     APInt Shift(Sz, 1);
    2681           0 :     Shift <<= J;
    2682           0 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
    2683           0 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
    2684             :   }
    2685             : 
    2686           0 :   return Tmp;
    2687             : }
    2688             : 
    2689             : /// Open code the operations for BSWAP of the specified operation.
    2690         131 : SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
    2691         131 :   EVT VT = Op.getValueType();
    2692         262 :   EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2693         131 :   SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
    2694         262 :   switch (VT.getSimpleVT().getScalarType().SimpleTy) {
    2695           0 :   default: llvm_unreachable("Unhandled Expand type in BSWAP!");
    2696          68 :   case MVT::i16:
    2697         136 :     Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2698         136 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2699         136 :     return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
    2700          25 :   case MVT::i32:
    2701          50 :     Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2702          50 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2703          50 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2704          50 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2705          50 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
    2706          50 :                        DAG.getConstant(0xFF0000, dl, VT));
    2707          50 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
    2708          50 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    2709          50 :     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    2710          50 :     return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
    2711          38 :   case MVT::i64:
    2712          76 :     Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    2713          76 :     Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
    2714          76 :     Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2715          76 :     Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2716          76 :     Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2717          76 :     Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2718          76 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
    2719          76 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    2720          76 :     Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
    2721          76 :                        DAG.getConstant(255ULL<<48, dl, VT));
    2722          76 :     Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
    2723          76 :                        DAG.getConstant(255ULL<<40, dl, VT));
    2724          76 :     Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
    2725          76 :                        DAG.getConstant(255ULL<<32, dl, VT));
    2726          76 :     Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
    2727          76 :                        DAG.getConstant(255ULL<<24, dl, VT));
    2728          76 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
    2729          76 :                        DAG.getConstant(255ULL<<16, dl, VT));
    2730          76 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
    2731          76 :                        DAG.getConstant(255ULL<<8 , dl, VT));
    2732          76 :     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
    2733          76 :     Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
    2734          76 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    2735          76 :     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    2736          76 :     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
    2737          76 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
    2738          76 :     return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
    2739             :   }
    2740             : }
    2741             : 
    2742             : /// Expand the specified bitcount instruction into operations.
    2743         348 : SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
    2744             :                                              const SDLoc &dl) {
    2745         348 :   switch (Opc) {
    2746           0 :   default: llvm_unreachable("Cannot expand this yet!");
    2747          65 :   case ISD::CTPOP: {
    2748          65 :     EVT VT = Op.getValueType();
    2749         130 :     EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2750          65 :     unsigned Len = VT.getSizeInBits();
    2751             : 
    2752             :     assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
    2753             :            "CTPOP not implemented for this type.");
    2754             : 
    2755             :     // This is the "best" algorithm from
    2756             :     // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
    2757             : 
    2758         260 :     SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
    2759          65 :                                      dl, VT);
    2760         260 :     SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
    2761          65 :                                      dl, VT);
    2762         260 :     SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
    2763          65 :                                      dl, VT);
    2764         260 :     SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
    2765          65 :                                      dl, VT);
    2766             : 
    2767             :     // v = v - ((v >> 1) & 0x55555555...)
    2768         130 :     Op = DAG.getNode(ISD::SUB, dl, VT, Op,
    2769             :                      DAG.getNode(ISD::AND, dl, VT,
    2770             :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2771             :                                              DAG.getConstant(1, dl, ShVT)),
    2772         130 :                                  Mask55));
    2773             :     // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
    2774         130 :     Op = DAG.getNode(ISD::ADD, dl, VT,
    2775          65 :                      DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
    2776             :                      DAG.getNode(ISD::AND, dl, VT,
    2777             :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2778             :                                              DAG.getConstant(2, dl, ShVT)),
    2779         195 :                                  Mask33));
    2780             :     // v = (v + (v >> 4)) & 0x0F0F0F0F...
    2781         130 :     Op = DAG.getNode(ISD::AND, dl, VT,
    2782             :                      DAG.getNode(ISD::ADD, dl, VT, Op,
    2783             :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2784             :                                              DAG.getConstant(4, dl, ShVT))),
    2785         130 :                      Mask0F);
    2786             :     // v = (v * 0x01010101...) >> (Len - 8)
    2787         130 :     Op = DAG.getNode(ISD::SRL, dl, VT,
    2788          65 :                      DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
    2789         195 :                      DAG.getConstant(Len - 8, dl, ShVT));
    2790             : 
    2791          65 :     return Op;
    2792             :   }
    2793         109 :   case ISD::CTLZ_ZERO_UNDEF:
    2794             :     // This trivially expands to CTLZ.
    2795         218 :     return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
    2796          61 :   case ISD::CTLZ: {
    2797          61 :     EVT VT = Op.getValueType();
    2798          61 :     unsigned Len = VT.getSizeInBits();
    2799             : 
    2800          61 :     if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
    2801          40 :       EVT SetCCVT = getSetCCResultType(VT);
    2802          80 :       SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
    2803          40 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    2804          40 :       SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
    2805          40 :       return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
    2806          40 :                          DAG.getConstant(Len, dl, VT), CTLZ);
    2807             :     }
    2808             : 
    2809             :     // for now, we do this:
    2810             :     // x = x | (x >> 1);
    2811             :     // x = x | (x >> 2);
    2812             :     // ...
    2813             :     // x = x | (x >>16);
    2814             :     // x = x | (x >>32); // for 64-bit input
    2815             :     // return popcount(~x);
    2816             :     //
    2817             :     // Ref: "Hacker's Delight" by Henry Warren
    2818          42 :     EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2819         243 :     for (unsigned i = 0; (1U << i) <= (Len / 2); ++i) {
    2820         111 :       SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
    2821         222 :       Op = DAG.getNode(ISD::OR, dl, VT, Op,
    2822         222 :                        DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
    2823             :     }
    2824          21 :     Op = DAG.getNOT(dl, Op, VT);
    2825          42 :     return DAG.getNode(ISD::CTPOP, dl, VT, Op);
    2826             :   }
    2827          83 :   case ISD::CTTZ_ZERO_UNDEF:
    2828             :     // This trivially expands to CTTZ.
    2829         166 :     return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
    2830          30 :   case ISD::CTTZ: {
    2831          30 :     EVT VT = Op.getValueType();
    2832          30 :     unsigned Len = VT.getSizeInBits();
    2833             : 
    2834          30 :     if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
    2835          17 :       EVT SetCCVT = getSetCCResultType(VT);
    2836          34 :       SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
    2837          17 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    2838          17 :       SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
    2839          17 :       return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
    2840          17 :                          DAG.getConstant(Len, dl, VT), CTTZ);
    2841             :     }
    2842             : 
    2843             :     // for now, we use: { return popcount(~x & (x - 1)); }
    2844             :     // unless the target has ctlz but not ctpop, in which case we use:
    2845             :     // { return 32 - nlz(~x & (x-1)); }
    2846             :     // Ref: "Hacker's Delight" by Henry Warren
    2847          13 :     SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
    2848          13 :                                DAG.getNOT(dl, Op, VT),
    2849             :                                DAG.getNode(ISD::SUB, dl, VT, Op,
    2850          26 :                                            DAG.getConstant(1, dl, VT)));
    2851             :     // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
    2852          13 :     if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
    2853             :         TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
    2854           5 :       return DAG.getNode(ISD::SUB, dl, VT,
    2855           5 :                          DAG.getConstant(VT.getSizeInBits(), dl, VT),
    2856          10 :                          DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
    2857          16 :     return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
    2858             :   }
    2859             :   }
    2860             : }
    2861             : 
    2862      224738 : bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
    2863             :   DEBUG(dbgs() << "Trying to expand node\n");
    2864             :   SmallVector<SDValue, 8> Results;
    2865             :   SDLoc dl(Node);
    2866      224738 :   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
    2867             :   bool NeedInvert;
    2868      224738 :   switch (Node->getOpcode()) {
    2869         348 :   case ISD::CTPOP:
    2870             :   case ISD::CTLZ:
    2871             :   case ISD::CTLZ_ZERO_UNDEF:
    2872             :   case ISD::CTTZ:
    2873             :   case ISD::CTTZ_ZERO_UNDEF:
    2874         348 :     Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
    2875         348 :     Results.push_back(Tmp1);
    2876         348 :     break;
    2877          92 :   case ISD::BITREVERSE:
    2878          92 :     Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
    2879          92 :     break;
    2880         131 :   case ISD::BSWAP:
    2881         131 :     Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
    2882         131 :     break;
    2883           0 :   case ISD::FRAMEADDR:
    2884             :   case ISD::RETURNADDR:
    2885             :   case ISD::FRAME_TO_ARGS_OFFSET:
    2886           0 :     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
    2887           0 :     break;
    2888           6 :   case ISD::EH_DWARF_CFA: {
    2889           6 :     SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
    2890           6 :                                         TLI.getPointerTy(DAG.getDataLayout()));
    2891           6 :     SDValue Offset = DAG.getNode(ISD::ADD, dl,
    2892             :                                  CfaArg.getValueType(),
    2893             :                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
    2894             :                                              CfaArg.getValueType()),
    2895          12 :                                  CfaArg);
    2896           6 :     SDValue FA = DAG.getNode(
    2897           6 :         ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
    2898          12 :         DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
    2899          12 :     Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
    2900           6 :                                   FA, Offset));
    2901             :     break;
    2902             :   }
    2903           1 :   case ISD::FLT_ROUNDS_:
    2904           2 :     Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
    2905           1 :     break;
    2906         276 :   case ISD::EH_RETURN:
    2907             :   case ISD::EH_LABEL:
    2908             :   case ISD::PREFETCH:
    2909             :   case ISD::VAEND:
    2910             :   case ISD::EH_SJLJ_LONGJMP:
    2911             :     // If the target didn't expand these, there's nothing to do, so just
    2912             :     // preserve the chain and be done.
    2913         276 :     Results.push_back(Node->getOperand(0));
    2914         276 :     break;
    2915           3 :   case ISD::READCYCLECOUNTER:
    2916             :     // If the target didn't expand this, just return 'zero' and preserve the
    2917             :     // chain.
    2918           3 :     Results.append(Node->getNumValues() - 1,
    2919           9 :                    DAG.getConstant(0, dl, Node->getValueType(0)));
    2920           3 :     Results.push_back(Node->getOperand(0));
    2921           3 :     break;
    2922           0 :   case ISD::EH_SJLJ_SETJMP:
    2923             :     // If the target didn't expand this, just return 'zero' and preserve the
    2924             :     // chain.
    2925           0 :     Results.push_back(DAG.getConstant(0, dl, MVT::i32));
    2926           0 :     Results.push_back(Node->getOperand(0));
    2927           0 :     break;
    2928           6 :   case ISD::ATOMIC_LOAD: {
    2929             :     // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
    2930          12 :     SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
    2931          12 :     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
    2932           6 :     SDValue Swap = DAG.getAtomicCmpSwap(
    2933             :         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
    2934             :         Node->getOperand(0), Node->getOperand(1), Zero, Zero,
    2935           6 :         cast<AtomicSDNode>(Node)->getMemOperand());
    2936           6 :     Results.push_back(Swap.getValue(0));
    2937           6 :     Results.push_back(Swap.getValue(1));
    2938             :     break;
    2939             :   }
    2940          14 :   case ISD::ATOMIC_STORE: {
    2941             :     // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
    2942          14 :     SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
    2943             :                                  cast<AtomicSDNode>(Node)->getMemoryVT(),
    2944             :                                  Node->getOperand(0),
    2945             :                                  Node->getOperand(1), Node->getOperand(2),
    2946          14 :                                  cast<AtomicSDNode>(Node)->getMemOperand());
    2947          14 :     Results.push_back(Swap.getValue(1));
    2948             :     break;
    2949             :   }
    2950         923 :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
    2951             :     // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
    2952             :     // splits out the success value as a comparison. Expanding the resulting
    2953             :     // ATOMIC_CMP_SWAP will produce a libcall.
    2954        1846 :     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
    2955         923 :     SDValue Res = DAG.getAtomicCmpSwap(
    2956             :         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
    2957             :         Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
    2958         923 :         Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
    2959             : 
    2960         923 :     SDValue ExtRes = Res;
    2961         923 :     SDValue LHS = Res;
    2962         923 :     SDValue RHS = Node->getOperand(1);
    2963             : 
    2964         923 :     EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
    2965         923 :     EVT OuterType = Node->getValueType(0);
    2966         923 :     switch (TLI.getExtendForAtomicOps()) {
    2967          55 :     case ISD::SIGN_EXTEND:
    2968         110 :       LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
    2969         110 :                         DAG.getValueType(AtomicType));
    2970         110 :       RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
    2971         110 :                         Node->getOperand(2), DAG.getValueType(AtomicType));
    2972          55 :       ExtRes = LHS;
    2973          55 :       break;
    2974         817 :     case ISD::ZERO_EXTEND:
    2975        1634 :       LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
    2976        1634 :                         DAG.getValueType(AtomicType));
    2977        1634 :       RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
    2978         817 :       ExtRes = LHS;
    2979         817 :       break;
    2980          51 :     case ISD::ANY_EXTEND:
    2981          51 :       LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
    2982         102 :       RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
    2983          51 :       break;
    2984           0 :     default:
    2985           0 :       llvm_unreachable("Invalid atomic op extension");
    2986             :     }
    2987             : 
    2988             :     SDValue Success =
    2989        1846 :         DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
    2990             : 
    2991         923 :     Results.push_back(ExtRes.getValue(0));
    2992         923 :     Results.push_back(Success);
    2993         923 :     Results.push_back(Res.getValue(1));
    2994             :     break;
    2995             :   }
    2996         139 :   case ISD::DYNAMIC_STACKALLOC:
    2997         139 :     ExpandDYNAMIC_STACKALLOC(Node, Results);
    2998         139 :     break;
    2999             :   case ISD::MERGE_VALUES:
    3000      742353 :     for (unsigned i = 0; i < Node->getNumValues(); i++)
    3001      445390 :       Results.push_back(Node->getOperand(i));
    3002             :     break;
    3003         453 :   case ISD::UNDEF: {
    3004         453 :     EVT VT = Node->getValueType(0);
    3005         453 :     if (VT.isInteger())
    3006         335 :       Results.push_back(DAG.getConstant(0, dl, VT));
    3007             :     else {
    3008             :       assert(VT.isFloatingPoint() && "Unknown value type!");
    3009         118 :       Results.push_back(DAG.getConstantFP(0, dl, VT));
    3010             :     }
    3011             :     break;
    3012             :   }
    3013         400 :   case ISD::FP_ROUND:
    3014             :   case ISD::BITCAST:
    3015         400 :     Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
    3016             :                             Node->getValueType(0), dl);
    3017         400 :     Results.push_back(Tmp1);
    3018         400 :     break;
    3019           0 :   case ISD::FP_EXTEND:
    3020           0 :     Tmp1 = EmitStackConvert(Node->getOperand(0),
    3021             :                             Node->getOperand(0).getValueType(),
    3022             :                             Node->getValueType(0), dl);
    3023           0 :     Results.push_back(Tmp1);
    3024           0 :     break;
    3025         769 :   case ISD::SIGN_EXTEND_INREG: {
    3026         769 :     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
    3027         769 :     EVT VT = Node->getValueType(0);
    3028             : 
    3029             :     // An in-register sign-extend of a boolean is a negation:
    3030             :     // 'true' (1) sign-extended is -1.
    3031             :     // 'false' (0) sign-extended is 0.
    3032             :     // However, we must mask the high bits of the source operand because the
    3033             :     // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
    3034             : 
    3035             :     // TODO: Do this for vectors too?
    3036         769 :     if (ExtraVT.getSizeInBits() == 1) {
    3037         235 :       SDValue One = DAG.getConstant(1, dl, VT);
    3038         470 :       SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
    3039         235 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    3040         470 :       SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
    3041         235 :       Results.push_back(Neg);
    3042             :       break;
    3043             :     }
    3044             : 
    3045             :     // NOTE: we could fall back on load/store here too for targets without
    3046             :     // SRA.  However, it is doubtful that any exist.
    3047        1068 :     EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    3048             :     unsigned BitsDiff = VT.getScalarSizeInBits() -
    3049         534 :                         ExtraVT.getScalarSizeInBits();
    3050         534 :     SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
    3051        1068 :     Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
    3052         534 :                        Node->getOperand(0), ShiftCst);
    3053        1068 :     Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
    3054         534 :     Results.push_back(Tmp1);
    3055         534 :     break;
    3056             :   }
    3057           0 :   case ISD::FP_ROUND_INREG: {
    3058             :     // The only way we can lower this is to turn it into a TRUNCSTORE,
    3059             :     // EXTLOAD pair, targeting a temporary location (a stack slot).
    3060             : 
    3061             :     // NOTE: there is a choice here between constantly creating new stack
    3062             :     // slots and always reusing the same one.  We currently always create
    3063             :     // new ones, as reuse may inhibit scheduling.
    3064           0 :     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
    3065           0 :     Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
    3066             :                             Node->getValueType(0), dl);
    3067           0 :     Results.push_back(Tmp1);
    3068             :     break;
    3069             :   }
    3070         137 :   case ISD::SINT_TO_FP:
    3071             :   case ISD::UINT_TO_FP:
    3072         137 :     Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
    3073         137 :                                 Node->getOperand(0), Node->getValueType(0), dl);
    3074         137 :     Results.push_back(Tmp1);
    3075         137 :     break;
    3076          60 :   case ISD::FP_TO_SINT:
    3077          60 :     if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
    3078          60 :       Results.push_back(Tmp1);
    3079             :     break;
    3080         192 :   case ISD::FP_TO_UINT: {
    3081         192 :     SDValue True, False;
    3082         192 :     EVT VT =  Node->getOperand(0).getValueType();
    3083         192 :     EVT NVT = Node->getValueType(0);
    3084             :     APFloat apf(DAG.EVTToAPFloatSemantics(VT),
    3085         384 :                 APInt::getNullValue(VT.getSizeInBits()));
    3086         192 :     APInt x = APInt::getSignMask(NVT.getSizeInBits());
    3087         192 :     (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
    3088         192 :     Tmp1 = DAG.getConstantFP(apf, dl, VT);
    3089         384 :     Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
    3090             :                         Node->getOperand(0),
    3091         192 :                         Tmp1, ISD::SETLT);
    3092         384 :     True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
    3093             :     // TODO: Should any fast-math-flags be set for the FSUB?
    3094         384 :     False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
    3095             :                         DAG.getNode(ISD::FSUB, dl, VT,
    3096         384 :                                     Node->getOperand(0), Tmp1));
    3097         384 :     False = DAG.getNode(ISD::XOR, dl, NVT, False,
    3098         384 :                         DAG.getConstant(x, dl, NVT));
    3099         192 :     Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
    3100         192 :     Results.push_back(Tmp1);
    3101             :     break;
    3102             :   }
    3103          24 :   case ISD::VAARG:
    3104          24 :     Results.push_back(DAG.expandVAArg(Node));
    3105          24 :     Results.push_back(Results[0].getValue(1));
    3106          24 :     break;
    3107           1 :   case ISD::VACOPY:
    3108           1 :     Results.push_back(DAG.expandVACopy(Node));
    3109           1 :     break;
    3110        2670 :   case ISD::EXTRACT_VECTOR_ELT:
    3111        2670 :     if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
    3112             :       // This must be an access of the only element.  Return it.
    3113         120 :       Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
    3114          60 :                          Node->getOperand(0));
    3115             :     else
    3116        2610 :       Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
    3117        2670 :     Results.push_back(Tmp1);
    3118        2670 :     break;
    3119             :   case ISD::EXTRACT_SUBVECTOR:
    3120           0 :     Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
    3121           0 :     break;
    3122             :   case ISD::INSERT_SUBVECTOR:
    3123           0 :     Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
    3124           0 :     break;
    3125           0 :   case ISD::CONCAT_VECTORS:
    3126           0 :     Results.push_back(ExpandVectorBuildThroughStack(Node));
    3127           0 :     break;
    3128          18 :   case ISD::SCALAR_TO_VECTOR:
    3129          18 :     Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
    3130          18 :     break;
    3131         589 :   case ISD::INSERT_VECTOR_ELT:
    3132         589 :     Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
    3133             :                                               Node->getOperand(1),
    3134             :                                               Node->getOperand(2), dl));
    3135         589 :     break;
    3136             :   case ISD::VECTOR_SHUFFLE: {
    3137             :     SmallVector<int, 32> NewMask;
    3138             :     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
    3139             : 
    3140         220 :     EVT VT = Node->getValueType(0);
    3141         220 :     EVT EltVT = VT.getVectorElementType();
    3142         220 :     SDValue Op0 = Node->getOperand(0);
    3143         220 :     SDValue Op1 = Node->getOperand(1);
    3144         220 :     if (!TLI.isTypeLegal(EltVT)) {
    3145          14 :       EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
    3146             : 
    3147             :       // BUILD_VECTOR operands are allowed to be wider than the element type.
    3148             :       // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
    3149             :       // it.
    3150           7 :       if (NewEltVT.bitsLT(EltVT)) {
    3151             :         // Convert shuffle node.
    3152             :         // If original node was v4i64 and the new EltVT is i32,
    3153             :         // cast operands to v8i32 and re-build the mask.
    3154             : 
    3155             :         // Calculate new VT, the size of the new VT should be equal to original.
    3156             :         EVT NewVT =
    3157           0 :             EVT::getVectorVT(*DAG.getContext(), NewEltVT,
    3158           0 :                              VT.getSizeInBits() / NewEltVT.getSizeInBits());
    3159             :         assert(NewVT.bitsEq(VT));
    3160             : 
    3161             :         // cast operands to new VT
    3162           0 :         Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
    3163           0 :         Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
    3164             : 
    3165             :         // Convert the shuffle mask
    3166             :         unsigned int factor =
    3167           0 :                          NewVT.getVectorNumElements()/VT.getVectorNumElements();
    3168             : 
    3169             :         // EltVT gets smaller
    3170             :         assert(factor > 0);
    3171             : 
    3172           0 :         for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
    3173           0 :           if (Mask[i] < 0) {
    3174           0 :             for (unsigned fi = 0; fi < factor; ++fi)
    3175           0 :               NewMask.push_back(Mask[i]);
    3176             :           }
    3177             :           else {
    3178           0 :             for (unsigned fi = 0; fi < factor; ++fi)
    3179           0 :               NewMask.push_back(Mask[i]*factor+fi);
    3180             :           }
    3181             :         }
    3182             :         Mask = NewMask;
    3183           0 :         VT = NewVT;
    3184             :       }
    3185           7 :       EltVT = NewEltVT;
    3186             :     }
    3187         220 :     unsigned NumElems = VT.getVectorNumElements();
    3188             :     SmallVector<SDValue, 16> Ops;
    3189        1628 :     for (unsigned i = 0; i != NumElems; ++i) {
    3190        1437 :       if (Mask[i] < 0) {
    3191          29 :         Ops.push_back(DAG.getUNDEF(EltVT));
    3192          29 :         continue;
    3193             :       }
    3194         675 :       unsigned Idx = Mask[i];
    3195         675 :       if (Idx < NumElems)
    3196         774 :         Ops.push_back(DAG.getNode(
    3197             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
    3198        1548 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3199             :       else
    3200         576 :         Ops.push_back(DAG.getNode(
    3201             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
    3202         288 :             DAG.getConstant(Idx - NumElems, dl,
    3203        1152 :                             TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3204             :     }
    3205             : 
    3206         440 :     Tmp1 = DAG.getBuildVector(VT, dl, Ops);
    3207             :     // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
    3208         440 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
    3209         220 :     Results.push_back(Tmp1);
    3210             :     break;
    3211             :   }
    3212         388 :   case ISD::EXTRACT_ELEMENT: {
    3213         388 :     EVT OpTy = Node->getOperand(0).getValueType();
    3214         388 :     if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
    3215             :       // 1 -> Hi
    3216         348 :       Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
    3217         174 :                          DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
    3218         174 :                                          TLI.getShiftAmountTy(
    3219             :                                              Node->getOperand(0).getValueType(),
    3220         696 :                                              DAG.getDataLayout())));
    3221         348 :       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
    3222             :     } else {
    3223             :       // 0 -> Lo
    3224         428 :       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
    3225         214 :                          Node->getOperand(0));
    3226             :     }
    3227         388 :     Results.push_back(Tmp1);
    3228             :     break;
    3229             :   }
    3230         111 :   case ISD::STACKSAVE:
    3231             :     // Expand to CopyFromReg if the target set
    3232             :     // StackPointerRegisterToSaveRestore.
    3233         111 :     if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
    3234         222 :       Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
    3235             :                                            Node->getValueType(0)));
    3236         111 :       Results.push_back(Results[0].getValue(1));
    3237             :     } else {
    3238           0 :       Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
    3239           0 :       Results.push_back(Node->getOperand(0));
    3240             :     }
    3241             :     break;
    3242          21 :   case ISD::STACKRESTORE:
    3243             :     // Expand to CopyToReg if the target set
    3244             :     // StackPointerRegisterToSaveRestore.
    3245          21 :     if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
    3246          42 :       Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
    3247             :                                          Node->getOperand(1)));
    3248             :     } else {
    3249           0 :       Results.push_back(Node->getOperand(0));
    3250             :     }
    3251             :     break;
    3252           0 :   case ISD::GET_DYNAMIC_AREA_OFFSET:
    3253           0 :     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
    3254           0 :     Results.push_back(Results[0].getValue(0));
    3255           0 :     break;
    3256          42 :   case ISD::FCOPYSIGN:
    3257          42 :     Results.push_back(ExpandFCOPYSIGN(Node));
    3258          42 :     break;
    3259           1 :   case ISD::FNEG:
    3260             :     // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
    3261           2 :     Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
    3262             :     // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
    3263           2 :     Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
    3264           1 :                        Node->getOperand(0));
    3265           1 :     Results.push_back(Tmp1);
    3266           1 :     break;
    3267           2 :   case ISD::FABS:
    3268           2 :     Results.push_back(ExpandFABS(Node));
    3269           2 :     break;
    3270           0 :   case ISD::SMIN:
    3271             :   case ISD::SMAX:
    3272             :   case ISD::UMIN:
    3273             :   case ISD::UMAX: {
    3274             :     // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
    3275             :     ISD::CondCode Pred;
    3276           0 :     switch (Node->getOpcode()) {
    3277           0 :     default: llvm_unreachable("How did we get here?");
    3278             :     case ISD::SMAX: Pred = ISD::SETGT; break;
    3279           0 :     case ISD::SMIN: Pred = ISD::SETLT; break;
    3280           0 :     case ISD::UMAX: Pred = ISD::SETUGT; break;
    3281           0 :     case ISD::UMIN: Pred = ISD::SETULT; break;
    3282             :     }
    3283           0 :     Tmp1 = Node->getOperand(0);
    3284           0 :     Tmp2 = Node->getOperand(1);
    3285           0 :     Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
    3286           0 :     Results.push_back(Tmp1);
    3287           0 :     break;
    3288             :   }
    3289             : 
    3290         287 :   case ISD::FSIN:
    3291             :   case ISD::FCOS: {
    3292         287 :     EVT VT = Node->getValueType(0);
    3293             :     // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
    3294             :     // fcos which share the same operand and both are used.
    3295         554 :     if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
    3296             :          isSinCosLibcallAvailable(Node, TLI))
    3297         139 :         && useSinCos(Node)) {
    3298          78 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3299         156 :       Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
    3300          78 :       if (Node->getOpcode() == ISD::FCOS)
    3301          39 :         Tmp1 = Tmp1.getValue(1);
    3302          78 :       Results.push_back(Tmp1);
    3303             :     }
    3304             :     break;
    3305             :   }
    3306           0 :   case ISD::FMAD:
    3307           0 :     llvm_unreachable("Illegal fmad should never be formed");
    3308             : 
    3309             :   case ISD::FP16_TO_FP:
    3310             :     if (Node->getValueType(0) != MVT::f32) {
    3311             :       // We can extend to types bigger than f32 in two steps without changing
    3312             :       // the result. Since "f16 -> f32" is much more commonly available, give
    3313             :       // CodeGen the option of emitting that before resorting to a libcall.
    3314             :       SDValue Res =
    3315           0 :           DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
    3316           0 :       Results.push_back(
    3317           0 :           DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
    3318             :     }
    3319             :     break;
    3320         471 :   case ISD::FP_TO_FP16:
    3321             :     DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
    3322         471 :     if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
    3323          15 :       SDValue Op = Node->getOperand(0);
    3324             :       MVT SVT = Op.getSimpleValueType();
    3325          15 :       if ((SVT == MVT::f64 || SVT == MVT::f80) &&
    3326          15 :           TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
    3327             :         // Under fastmath, we can expand this node into a fround followed by
    3328             :         // a float-half conversion.
    3329          13 :         SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
    3330          26 :                                        DAG.getIntPtrConstant(0, dl));
    3331          13 :         Results.push_back(
    3332          39 :             DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
    3333             :       }
    3334             :     }
    3335             :     break;
    3336             :   case ISD::ConstantFP: {
    3337             :     ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
    3338             :     // Check to see if this FP immediate is already legal.
    3339             :     // If this is a legal constant, turn it into a TargetConstantFP node.
    3340       10726 :     if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
    3341        2936 :       Results.push_back(ExpandConstantFP(CFP, true));
    3342             :     break;
    3343             :   }
    3344             :   case ISD::Constant: {
    3345             :     ConstantSDNode *CP = cast<ConstantSDNode>(Node);
    3346           0 :     Results.push_back(ExpandConstant(CP));
    3347           0 :     break;
    3348             :   }
    3349         168 :   case ISD::FSUB: {
    3350         168 :     EVT VT = Node->getValueType(0);
    3351         168 :     if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
    3352             :         TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
    3353         164 :       const SDNodeFlags Flags = Node->getFlags();
    3354         328 :       Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
    3355         328 :       Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
    3356         164 :       Results.push_back(Tmp1);
    3357             :     }
    3358             :     break;
    3359             :   }
    3360           0 :   case ISD::SUB: {
    3361           0 :     EVT VT = Node->getValueType(0);
    3362             :     assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
    3363             :            TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
    3364             :            "Don't know how to expand this subtraction!");
    3365           0 :     Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
    3366           0 :                DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
    3367           0 :                                VT));
    3368           0 :     Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
    3369           0 :     Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
    3370             :     break;
    3371             :   }
    3372         988 :   case ISD::UREM:
    3373             :   case ISD::SREM: {
    3374         988 :     EVT VT = Node->getValueType(0);
    3375             :     bool isSigned = Node->getOpcode() == ISD::SREM;
    3376         988 :     unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
    3377         988 :     unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3378         988 :     Tmp2 = Node->getOperand(0);
    3379         988 :     Tmp3 = Node->getOperand(1);
    3380         988 :     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
    3381         802 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3382         802 :       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
    3383         802 :       Results.push_back(Tmp1);
    3384         186 :     } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
    3385             :       // X % Y -> X-X/Y*Y
    3386         312 :       Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
    3387         312 :       Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
    3388         312 :       Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
    3389         156 :       Results.push_back(Tmp1);
    3390             :     }
    3391             :     break;
    3392             :   }
    3393         802 :   case ISD::UDIV:
    3394             :   case ISD::SDIV: {
    3395             :     bool isSigned = Node->getOpcode() == ISD::SDIV;
    3396         802 :     unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3397         802 :     EVT VT = Node->getValueType(0);
    3398         802 :     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
    3399         794 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3400        1588 :       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
    3401             :                          Node->getOperand(1));
    3402         794 :       Results.push_back(Tmp1);
    3403             :     }
    3404             :     break;
    3405             :   }
    3406         440 :   case ISD::MULHU:
    3407             :   case ISD::MULHS: {
    3408             :     unsigned ExpandOpcode =
    3409         440 :         Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
    3410         440 :     EVT VT = Node->getValueType(0);
    3411         440 :     SDVTList VTs = DAG.getVTList(VT, VT);
    3412             : 
    3413         880 :     Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
    3414             :                        Node->getOperand(1));
    3415         440 :     Results.push_back(Tmp1.getValue(1));
    3416             :     break;
    3417             :   }
    3418         570 :   case ISD::UMUL_LOHI:
    3419             :   case ISD::SMUL_LOHI: {
    3420         570 :     SDValue LHS = Node->getOperand(0);
    3421         570 :     SDValue RHS = Node->getOperand(1);
    3422             :     MVT VT = LHS.getSimpleValueType();
    3423             :     unsigned MULHOpcode =
    3424         570 :         Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
    3425             : 
    3426         570 :     if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
    3427         912 :       Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
    3428         912 :       Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
    3429         456 :       break;
    3430             :     }
    3431             : 
    3432             :     SmallVector<SDValue, 4> Halves;
    3433         114 :     EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
    3434             :     assert(TLI.isTypeLegal(HalfType));
    3435         456 :     if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
    3436             :                            HalfType, DAG,
    3437         114 :                            TargetLowering::MulExpansionKind::Always)) {
    3438         570 :       for (unsigned i = 0; i < 2; ++i) {
    3439         684 :         SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
    3440         684 :         SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
    3441         228 :         SDValue Shift = DAG.getConstant(
    3442             :             HalfType.getScalarSizeInBits(), dl,
    3443         684 :             TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
    3444         456 :         Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
    3445         456 :         Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
    3446             :       }
    3447             :       break;
    3448             :     }
    3449             :     break;
    3450             :   }
    3451         178 :   case ISD::MUL: {
    3452         178 :     EVT VT = Node->getValueType(0);
    3453         178 :     SDVTList VTs = DAG.getVTList(VT, VT);
    3454             :     // See if multiply or divide can be lowered using two-result operations.
    3455             :     // We just need the low half of the multiply; try both the signed
    3456             :     // and unsigned forms. If the target supports both SMUL_LOHI and
    3457             :     // UMUL_LOHI, form a preference by checking which forms of plain
    3458             :     // MULH it supports.
    3459         178 :     bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
    3460             :     bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
    3461             :     bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
    3462             :     bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
    3463             :     unsigned OpToUse = 0;
    3464         178 :     if (HasSMUL_LOHI && !HasMULHS) {
    3465             :       OpToUse = ISD::SMUL_LOHI;
    3466         171 :     } else if (HasUMUL_LOHI && !HasMULHU) {
    3467             :       OpToUse = ISD::UMUL_LOHI;
    3468         171 :     } else if (HasSMUL_LOHI) {
    3469             :       OpToUse = ISD::SMUL_LOHI;
    3470         171 :     } else if (HasUMUL_LOHI) {
    3471             :       OpToUse = ISD::UMUL_LOHI;
    3472             :     }
    3473             :     if (OpToUse) {
    3474          14 :       Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
    3475             :                                     Node->getOperand(1)));
    3476           7 :       break;
    3477             :     }
    3478             : 
    3479         171 :     SDValue Lo, Hi;
    3480         171 :     EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
    3481         171 :     if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
    3482             :         TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
    3483             :         TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
    3484         171 :         TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
    3485         177 :         TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
    3486         171 :                       TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
    3487         330 :       Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
    3488         330 :       Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
    3489             :       SDValue Shift =
    3490         330 :           DAG.getConstant(HalfType.getSizeInBits(), dl,
    3491         495 :                           TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
    3492         330 :       Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
    3493         330 :       Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
    3494             :     }
    3495             :     break;
    3496             :   }
    3497          35 :   case ISD::SADDO:
    3498             :   case ISD::SSUBO: {
    3499          35 :     SDValue LHS = Node->getOperand(0);
    3500          35 :     SDValue RHS = Node->getOperand(1);
    3501          35 :     SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
    3502             :                               ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
    3503          35 :                               LHS, RHS);
    3504          35 :     Results.push_back(Sum);
    3505          35 :     EVT ResultType = Node->getValueType(1);
    3506          35 :     EVT OType = getSetCCResultType(Node->getValueType(0));
    3507             : 
    3508          70 :     SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
    3509             : 
    3510             :     //   LHSSign -> LHS >= 0
    3511             :     //   RHSSign -> RHS >= 0
    3512             :     //   SumSign -> Sum >= 0
    3513             :     //
    3514             :     //   Add:
    3515             :     //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
    3516             :     //   Sub:
    3517             :     //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
    3518          35 :     SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
    3519          35 :     SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
    3520          35 :     SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
    3521             :                                       Node->getOpcode() == ISD::SADDO ?
    3522          35 :                                       ISD::SETEQ : ISD::SETNE);
    3523             : 
    3524          35 :     SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
    3525          35 :     SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
    3526             : 
    3527          70 :     SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
    3528          35 :     Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
    3529             :     break;
    3530             :   }
    3531          22 :   case ISD::UADDO:
    3532             :   case ISD::USUBO: {
    3533          22 :     SDValue LHS = Node->getOperand(0);
    3534          22 :     SDValue RHS = Node->getOperand(1);
    3535          22 :     SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
    3536             :                               ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
    3537          22 :                               LHS, RHS);
    3538          22 :     Results.push_back(Sum);
    3539             : 
    3540          22 :     EVT ResultType = Node->getValueType(1);
    3541          22 :     EVT SetCCType = getSetCCResultType(Node->getValueType(0));
    3542             :     ISD::CondCode CC
    3543          22 :       = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
    3544          22 :     SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
    3545             : 
    3546          22 :     Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
    3547             :     break;
    3548             :   }
    3549          12 :   case ISD::UMULO:
    3550             :   case ISD::SMULO: {
    3551          12 :     EVT VT = Node->getValueType(0);
    3552          12 :     EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
    3553          12 :     SDValue LHS = Node->getOperand(0);
    3554          12 :     SDValue RHS = Node->getOperand(1);
    3555          12 :     SDValue BottomHalf;
    3556          12 :     SDValue TopHalf;
    3557             :     static const unsigned Ops[2][3] =
    3558             :         { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
    3559             :           { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
    3560          12 :     bool isSigned = Node->getOpcode() == ISD::SMULO;
    3561          12 :     if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
    3562          10 :       BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
    3563          10 :       TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
    3564           7 :     } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
    3565           4 :       BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
    3566           2 :                                RHS);
    3567           2 :       TopHalf = BottomHalf.getValue(1);
    3568           5 :     } else if (TLI.isTypeLegal(WideVT)) {
    3569           0 :       LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
    3570           0 :       RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
    3571           0 :       Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
    3572           0 :       BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
    3573           0 :                                DAG.getIntPtrConstant(0, dl));
    3574           0 :       TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
    3575           0 :                             DAG.getIntPtrConstant(1, dl));
    3576             :     } else {
    3577             :       // We can fall back to a libcall with an illegal type for the MUL if we
    3578             :       // have a libcall big enough.
    3579             :       // Also, we can fall back to a division in some cases, but that's a big
    3580             :       // performance hit in the general case.
    3581             :       RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    3582             :       if (WideVT == MVT::i16)
    3583             :         LC = RTLIB::MUL_I16;
    3584             :       else if (WideVT == MVT::i32)
    3585             :         LC = RTLIB::MUL_I32;
    3586             :       else if (WideVT == MVT::i64)
    3587             :         LC = RTLIB::MUL_I64;
    3588             :       else if (WideVT == MVT::i128)
    3589             :         LC = RTLIB::MUL_I128;
    3590             :       assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
    3591             : 
    3592             :       SDValue HiLHS;
    3593             :       SDValue HiRHS;
    3594           5 :       if (isSigned) {
    3595             :         // The high part is obtained by SRA'ing all but one of the bits of low
    3596             :         // part.
    3597           1 :         unsigned LoSize = VT.getSizeInBits();
    3598           1 :         HiLHS =
    3599           3 :             DAG.getNode(ISD::SRA, dl, VT, LHS,
    3600           1 :                         DAG.getConstant(LoSize - 1, dl,
    3601           2 :                                         TLI.getPointerTy(DAG.getDataLayout())));
    3602           1 :         HiRHS =
    3603           3 :             DAG.getNode(ISD::SRA, dl, VT, RHS,
    3604             :                         DAG.getConstant(LoSize - 1, dl,
    3605           2 :                                         TLI.getPointerTy(DAG.getDataLayout())));
    3606             :       } else {
    3607           4 :           HiLHS = DAG.getConstant(0, dl, VT);
    3608           4 :           HiRHS = DAG.getConstant(0, dl, VT);
    3609             :       }
    3610             : 
    3611             :       // Here we're passing the 2 arguments explicitly as 4 arguments that are
    3612             :       // pre-lowered to the correct types. This all depends upon WideVT not
    3613             :       // being a legal type for the architecture and thus has to be split to
    3614             :       // two arguments.
    3615             :       SDValue Ret;
    3616          10 :       if(DAG.getDataLayout().isLittleEndian()) {
    3617             :         // Halves of WideVT are packed into registers in different order
    3618             :         // depending on platform endianness. This is usually handled by
    3619             :         // the C calling convention, but we can't defer to it in
    3620             :         // the legalizer.
    3621           5 :         SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
    3622           5 :         Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
    3623             :       } else {
    3624           0 :         SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
    3625           0 :         Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
    3626             :       }
    3627             :       assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
    3628             :              "Ret value is a collection of constituent nodes holding result.");
    3629           5 :       BottomHalf = Ret.getOperand(0);
    3630           5 :       TopHalf = Ret.getOperand(1);
    3631             :     }
    3632             : 
    3633          12 :     if (isSigned) {
    3634          12 :       Tmp1 = DAG.getConstant(
    3635           6 :           VT.getSizeInBits() - 1, dl,
    3636          18 :           TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
    3637          12 :       Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
    3638          12 :       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
    3639           6 :                              ISD::SETNE);
    3640             :     } else {
    3641          12 :       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
    3642           6 :                              DAG.getConstant(0, dl, VT), ISD::SETNE);
    3643             :     }
    3644             : 
    3645             :     // Truncate the result if SetCC returns a larger type than needed.
    3646          12 :     EVT RType = Node->getValueType(1);
    3647          12 :     if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
    3648           2 :       TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
    3649             : 
    3650             :     assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
    3651             :            "Unexpected result type for S/UMULO legalization");
    3652             : 
    3653          12 :     Results.push_back(BottomHalf);
    3654          12 :     Results.push_back(TopHalf);
    3655             :     break;
    3656             :   }
    3657          22 :   case ISD::BUILD_PAIR: {
    3658          22 :     EVT PairTy = Node->getValueType(0);
    3659          44 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
    3660          44 :     Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
    3661          44 :     Tmp2 = DAG.getNode(
    3662             :         ISD::SHL, dl, PairTy, Tmp2,
    3663          22 :         DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
    3664          88 :                         TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
    3665          44 :     Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
    3666             :     break;
    3667             :   }
    3668         214 :   case ISD::SELECT:
    3669         214 :     Tmp1 = Node->getOperand(0);
    3670         214 :     Tmp2 = Node->getOperand(1);
    3671         214 :     Tmp3 = Node->getOperand(2);
    3672         214 :     if (Tmp1.getOpcode() == ISD::SETCC) {
    3673         312 :       Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
    3674             :                              Tmp2, Tmp3,
    3675             :                              cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
    3676             :     } else {
    3677         116 :       Tmp1 = DAG.getSelectCC(dl, Tmp1,
    3678             :                              DAG.getConstant(0, dl, Tmp1.getValueType()),
    3679          58 :                              Tmp2, Tmp3, ISD::SETNE);
    3680             :     }
    3681         214 :     Results.push_back(Tmp1);
    3682         214 :     break;
    3683         159 :   case ISD::BR_JT: {
    3684         159 :     SDValue Chain = Node->getOperand(0);
    3685         159 :     SDValue Table = Node->getOperand(1);
    3686         159 :     SDValue Index = Node->getOperand(2);
    3687             : 
    3688         159 :     const DataLayout &TD = DAG.getDataLayout();
    3689             :     EVT PTy = TLI.getPointerTy(TD);
    3690             : 
    3691             :     unsigned EntrySize =
    3692         159 :       DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
    3693             : 
    3694         318 :     Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
    3695         318 :                         DAG.getConstant(EntrySize, dl, Index.getValueType()));
    3696         159 :     SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
    3697         159 :                                Index, Table);
    3698             : 
    3699         159 :     EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
    3700         159 :     SDValue LD = DAG.getExtLoad(
    3701             :         ISD::SEXTLOAD, dl, PTy, Chain, Addr,
    3702         159 :         MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
    3703         159 :     Addr = LD;
    3704         159 :     if (TLI.isJumpTableRelative()) {
    3705             :       // For PIC, the sequence is:
    3706             :       // BRIND(load(Jumptable + index) + RelocBase)
    3707             :       // RelocBase can be JumpTable, GOT or some sort of global base.
    3708         102 :       Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
    3709         102 :                           TLI.getPICJumpTableRelocBase(Table, DAG));
    3710             :     }
    3711         318 :     Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
    3712         159 :     Results.push_back(Tmp1);
    3713             :     break;
    3714             :   }
    3715         399 :   case ISD::BRCOND:
    3716             :     // Expand brcond's setcc into its constituent parts and create a BR_CC
    3717             :     // Node.
    3718         399 :     Tmp1 = Node->getOperand(0);
    3719         399 :     Tmp2 = Node->getOperand(1);
    3720         399 :     if (Tmp2.getOpcode() == ISD::SETCC) {
    3721           4 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
    3722             :                          Tmp1, Tmp2.getOperand(2),
    3723             :                          Tmp2.getOperand(0), Tmp2.getOperand(1),
    3724           2 :                          Node->getOperand(2));
    3725             :     } else {
    3726             :       // We test only the i1 bit.  Skip the AND if UNDEF or another AND.
    3727         397 :       if (Tmp2.isUndef() ||
    3728             :           (Tmp2.getOpcode() == ISD::AND &&
    3729          61 :            isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
    3730             :            dyn_cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
    3731          61 :         Tmp3 = Tmp2;
    3732             :       else
    3733         672 :         Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
    3734         672 :                            DAG.getConstant(1, dl, Tmp2.getValueType()));
    3735         794 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
    3736         397 :                          DAG.getCondCode(ISD::SETNE), Tmp3,
    3737             :                          DAG.getConstant(0, dl, Tmp3.getValueType()),
    3738        1191 :                          Node->getOperand(2));
    3739             :     }
    3740         399 :     Results.push_back(Tmp1);
    3741         399 :     break;
    3742        2286 :   case ISD::SETCC: {
    3743        2286 :     Tmp1 = Node->getOperand(0);
    3744        2286 :     Tmp2 = Node->getOperand(1);
    3745        2286 :     Tmp3 = Node->getOperand(2);
    3746             :     bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
    3747        2286 :                                            Tmp3, NeedInvert, dl);
    3748             : 
    3749        2286 :     if (Legalized) {
    3750             :       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
    3751             :       // condition code, create a new SETCC node.
    3752         283 :       if (Tmp3.getNode())
    3753         402 :         Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
    3754             :                            Tmp1, Tmp2, Tmp3);
    3755             : 
    3756             :       // If we expanded the SETCC by inverting the condition code, then wrap
    3757             :       // the existing SETCC in a NOT to restore the intended condition.
    3758         283 :       if (NeedInvert)
    3759          96 :         Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
    3760             : 
    3761         283 :       Results.push_back(Tmp1);
    3762         283 :       break;
    3763             :     }
    3764             : 
    3765             :     // Otherwise, SETCC for the given comparison type must be completely
    3766             :     // illegal; expand it into a SELECT_CC.
    3767        2003 :     EVT VT = Node->getValueType(0);
    3768             :     int TrueValue;
    3769        4006 :     switch (TLI.getBooleanContents(Tmp1.getValueType())) {
    3770         646 :     case TargetLowering::ZeroOrOneBooleanContent:
    3771             :     case TargetLowering::UndefinedBooleanContent:
    3772             :       TrueValue = 1;
    3773         646 :       break;
    3774        1357 :     case TargetLowering::ZeroOrNegativeOneBooleanContent:
    3775             :       TrueValue = -1;
    3776        1357 :       break;
    3777             :     }
    3778        4006 :     Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
    3779        2003 :                        DAG.getConstant(TrueValue, dl, VT),
    3780             :                        DAG.getConstant(0, dl, VT),
    3781        4006 :                        Tmp3);
    3782        2003 :     Results.push_back(Tmp1);
    3783        2003 :     break;
    3784             :   }
    3785        2156 :   case ISD::SELECT_CC: {
    3786        2156 :     Tmp1 = Node->getOperand(0);   // LHS
    3787        2156 :     Tmp2 = Node->getOperand(1);   // RHS
    3788        2156 :     Tmp3 = Node->getOperand(2);   // True
    3789        2156 :     Tmp4 = Node->getOperand(3);   // False
    3790        2156 :     EVT VT = Node->getValueType(0);
    3791        2156 :     SDValue CC = Node->getOperand(4);
    3792             :     ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
    3793             : 
    3794        2156 :     if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
    3795             :       // If the condition code is legal, then we need to expand this
    3796             :       // node using SETCC and SELECT.
    3797        1855 :       EVT CmpVT = Tmp1.getValueType();
    3798             :       assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
    3799             :              "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
    3800             :              "expanded.");
    3801             :       EVT CCVT =
    3802        3710 :           TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
    3803        1855 :       SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
    3804        1855 :       Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
    3805             :       break;
    3806             :     }
    3807             : 
    3808             :     // SELECT_CC is legal, so the condition code must not be.
    3809             :     bool Legalized = false;
    3810             :     // Try to legalize by inverting the condition.  This is for targets that
    3811             :     // might support an ordered version of a condition, but not the unordered
    3812             :     // version (or vice versa).
    3813         301 :     ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
    3814         602 :                                                Tmp1.getValueType().isInteger());
    3815         301 :     if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
    3816             :       // Use the new condition code and swap true and false
    3817             :       Legalized = true;
    3818         256 :       Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
    3819             :     } else {
    3820             :       // If The inverse is not legal, then try to swap the arguments using
    3821             :       // the inverse condition code.
    3822          45 :       ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
    3823          45 :       if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
    3824             :         // The swapped inverse condition is legal, so swap true and false,
    3825             :         // lhs and rhs.
    3826             :         Legalized = true;
    3827          12 :         Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
    3828             :       }
    3829             :     }
    3830             : 
    3831             :     if (!Legalized) {
    3832          33 :       Legalized = LegalizeSetCCCondCode(
    3833             :           getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
    3834             :           dl);
    3835             : 
    3836             :       assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
    3837             : 
    3838             :       // If we expanded the SETCC by inverting the condition code, then swap
    3839             :       // the True/False operands to match.
    3840          33 :       if (NeedInvert)
    3841             :         std::swap(Tmp3, Tmp4);
    3842             : 
    3843             :       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
    3844             :       // condition code, create a new SELECT_CC node.
    3845          33 :       if (CC.getNode()) {
    3846          26 :         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
    3847             :                            Tmp1, Tmp2, Tmp3, Tmp4, CC);
    3848             :       } else {
    3849          40 :         Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
    3850          20 :         CC = DAG.getCondCode(ISD::SETNE);
    3851          40 :         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
    3852             :                            Tmp2, Tmp3, Tmp4, CC);
    3853             :       }
    3854             :     }
    3855         301 :     Results.push_back(Tmp1);
    3856         301 :     break;
    3857             :   }
    3858           9 :   case ISD::BR_CC: {
    3859           9 :     Tmp1 = Node->getOperand(0);              // Chain
    3860           9 :     Tmp2 = Node->getOperand(2);              // LHS
    3861           9 :     Tmp3 = Node->getOperand(3);              // RHS
    3862           9 :     Tmp4 = Node->getOperand(1);              // CC
    3863             : 
    3864           9 :     bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
    3865           9 :         Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
    3866             :     (void)Legalized;
    3867             :     assert(Legalized && "Can't legalize BR_CC with legal condition!");
    3868             : 
    3869             :     // If we expanded the SETCC by inverting the condition code, then wrap
    3870             :     // the existing SETCC in a NOT to restore the intended condition.
    3871           9 :     if (NeedInvert)
    3872           0 :       Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
    3873             : 
    3874             :     // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
    3875             :     // node.
    3876           9 :     if (Tmp4.getNode()) {
    3877           0 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
    3878             :                          Tmp4, Tmp2, Tmp3, Node->getOperand(4));
    3879             :     } else {
    3880          18 :       Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
    3881           9 :       Tmp4 = DAG.getCondCode(ISD::SETNE);
    3882          18 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
    3883             :                          Tmp2, Tmp3, Node->getOperand(4));
    3884             :     }
    3885           9 :     Results.push_back(Tmp1);
    3886           9 :     break;
    3887             :   }
    3888       20573 :   case ISD::BUILD_VECTOR:
    3889       20573 :     Results.push_back(ExpandBUILD_VECTOR(Node));
    3890       20573 :     break;
    3891          10 :   case ISD::SRA:
    3892             :   case ISD::SRL:
    3893             :   case ISD::SHL: {
    3894             :     // Scalarize vector SRA/SRL/SHL.
    3895          10 :     EVT VT = Node->getValueType(0);
    3896             :     assert(VT.isVector() && "Unable to legalize non-vector shift");
    3897             :     assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
    3898          10 :     unsigned NumElem = VT.getVectorNumElements();
    3899             : 
    3900             :     SmallVector<SDValue, 8> Scalars;
    3901          50 :     for (unsigned Idx = 0; Idx < NumElem; Idx++) {
    3902          20 :       SDValue Ex = DAG.getNode(
    3903             :           ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
    3904          80 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3905          20 :       SDValue Sh = DAG.getNode(
    3906             :           ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
    3907          80 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3908          40 :       Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
    3909          40 :                                     VT.getScalarType(), Ex, Sh));
    3910             :     }
    3911             : 
    3912          20 :     SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
    3913          10 :     ReplaceNode(SDValue(Node, 0), Result);
    3914             :     break;
    3915             :   }
    3916             :   case ISD::GLOBAL_OFFSET_TABLE:
    3917             :   case ISD::GlobalAddress:
    3918             :   case ISD::GlobalTLSAddress:
    3919             :   case ISD::ExternalSymbol:
    3920             :   case ISD::ConstantPool:
    3921             :   case ISD::JumpTable:
    3922             :   case ISD::INTRINSIC_W_CHAIN:
    3923             :   case ISD::INTRINSIC_WO_CHAIN:
    3924             :   case ISD::INTRINSIC_VOID:
    3925             :     // FIXME: Custom lowering for these operations shouldn't return null!
    3926             :     break;
    3927             :   }
    3928             : 
    3929             :   // Replace the original node with the legalized result.
    3930      224738 :   if (Results.empty()) {
    3931             :     DEBUG(dbgs() << "Cannot expand node\n");
    3932             :     return false;
    3933             :   }
    3934             : 
    3935             :   DEBUG(dbgs() << "Succesfully expanded node\n");
    3936      188476 :   ReplaceNode(Node, Results.data());
    3937      188476 :   return true;
    3938             : }
    3939             : 
    3940       36334 : void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
    3941             :   DEBUG(dbgs() << "Trying to convert node to libcall\n");
    3942             :   SmallVector<SDValue, 8> Results;
    3943             :   SDLoc dl(Node);
    3944             :   // FIXME: Check flags on the node to see if we can use a finite call.
    3945       36334 :   bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
    3946             :   unsigned Opc = Node->getOpcode();
    3947       36334 :   switch (Opc) {
    3948             :   case ISD::ATOMIC_FENCE: {
    3949             :     // If the target didn't lower this, lower it to '__sync_synchronize()' call
    3950             :     // FIXME: handle "fence singlethread" more efficiently.
    3951             :     TargetLowering::ArgListTy Args;
    3952             : 
    3953          26 :     TargetLowering::CallLoweringInfo CLI(DAG);
    3954             :     CLI.setDebugLoc(dl)
    3955             :         .setChain(Node->getOperand(0))
    3956          13 :         .setLibCallee(
    3957          13 :             CallingConv::C, Type::getVoidTy(*DAG.getContext()),
    3958          13 :             DAG.getExternalSymbol("__sync_synchronize",
    3959             :                                   TLI.getPointerTy(DAG.getDataLayout())),
    3960          39 :             std::move(Args));
    3961             : 
    3962          13 :     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
    3963             : 
    3964          13 :     Results.push_back(CallResult.second);
    3965             :     break;
    3966             :   }
    3967             :   // By default, atomic intrinsics are marked Legal and lowered. Targets
    3968             :   // which don't support them directly, however, may want libcalls, in which
    3969             :   // case they mark them Expand, and we get here.
    3970             :   case ISD::ATOMIC_SWAP:
    3971             :   case ISD::ATOMIC_LOAD_ADD:
    3972             :   case ISD::ATOMIC_LOAD_SUB:
    3973             :   case ISD::ATOMIC_LOAD_AND:
    3974             :   case ISD::ATOMIC_LOAD_CLR:
    3975             :   case ISD::ATOMIC_LOAD_OR:
    3976             :   case ISD::ATOMIC_LOAD_XOR:
    3977             :   case ISD::ATOMIC_LOAD_NAND:
    3978             :   case ISD::ATOMIC_LOAD_MIN:
    3979             :   case ISD::ATOMIC_LOAD_MAX:
    3980             :   case ISD::ATOMIC_LOAD_UMIN:
    3981             :   case ISD::ATOMIC_LOAD_UMAX:
    3982             :   case ISD::ATOMIC_CMP_SWAP: {
    3983         309 :     MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
    3984         309 :     RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
    3985             :     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
    3986             : 
    3987         309 :     std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
    3988         309 :     Results.push_back(Tmp.first);
    3989         309 :     Results.push_back(Tmp.second);
    3990             :     break;
    3991             :   }
    3992             :   case ISD::TRAP: {
    3993             :     // If this operation is not supported, lower it to 'abort()' call
    3994             :     TargetLowering::ArgListTy Args;
    3995           4 :     TargetLowering::CallLoweringInfo CLI(DAG);
    3996             :     CLI.setDebugLoc(dl)
    3997             :         .setChain(Node->getOperand(0))
    3998           4 :         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
    3999           2 :                       DAG.getExternalSymbol(
    4000             :                           "abort", TLI.getPointerTy(DAG.getDataLayout())),
    4001           6 :                       std::move(Args));
    4002           2 :     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
    4003             : 
    4004           2 :     Results.push_back(CallResult.second);
    4005             :     break;
    4006             :   }
    4007          45 :   case ISD::FMINNUM:
    4008          45 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
    4009             :                                       RTLIB::FMIN_F80, RTLIB::FMIN_F128,
    4010             :                                       RTLIB::FMIN_PPCF128));
    4011          45 :     break;
    4012          47 :   case ISD::FMAXNUM:
    4013          47 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
    4014             :                                       RTLIB::FMAX_F80, RTLIB::FMAX_F128,
    4015             :                                       RTLIB::FMAX_PPCF128));
    4016          47 :     break;
    4017           4 :   case ISD::FSQRT:
    4018             :   case ISD::STRICT_FSQRT:
    4019           4 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
    4020             :                                       RTLIB::SQRT_F80, RTLIB::SQRT_F128,
    4021             :                                       RTLIB::SQRT_PPCF128));
    4022           4 :     break;
    4023         118 :   case ISD::FSIN:
    4024             :   case ISD::STRICT_FSIN:
    4025         118 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
    4026             :                                       RTLIB::SIN_F80, RTLIB::SIN_F128,
    4027             :                                       RTLIB::SIN_PPCF128));
    4028         118 :     break;
    4029          95 :   case ISD::FCOS:
    4030             :   case ISD::STRICT_FCOS:
    4031          95 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
    4032             :                                       RTLIB::COS_F80, RTLIB::COS_F128,
    4033             :                                       RTLIB::COS_PPCF128));
    4034          95 :     break;
    4035          30 :   case ISD::FSINCOS:
    4036             :     // Expand into sincos libcall.
    4037          30 :     ExpandSinCosLibCall(Node, Results);
    4038          30 :     break;
    4039          70 :   case ISD::FLOG:
    4040             :   case ISD::STRICT_FLOG:
    4041          81 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
    4042           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
    4043             :                                         RTLIB::LOG_FINITE_F64,
    4044             :                                         RTLIB::LOG_FINITE_F80,
    4045             :                                         RTLIB::LOG_FINITE_F128,
    4046             :                                         RTLIB::LOG_FINITE_PPCF128));
    4047             :     else
    4048          66 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
    4049             :                                         RTLIB::LOG_F80, RTLIB::LOG_F128,
    4050             :                                         RTLIB::LOG_PPCF128));
    4051             :     break;
    4052          73 :   case ISD::FLOG2:
    4053             :   case ISD::STRICT_FLOG2:
    4054          84 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
    4055           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
    4056             :                                         RTLIB::LOG2_FINITE_F64,
    4057             :                                         RTLIB::LOG2_FINITE_F80,
    4058             :                                         RTLIB::LOG2_FINITE_F128,
    4059             :                                         RTLIB::LOG2_FINITE_PPCF128));
    4060             :     else
    4061          69 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
    4062             :                                         RTLIB::LOG2_F80, RTLIB::LOG2_F128,
    4063             :                                         RTLIB::LOG2_PPCF128));
    4064             :     break;
    4065          74 :   case ISD::FLOG10:
    4066             :   case ISD::STRICT_FLOG10:
    4067          85 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
    4068           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
    4069             :                                         RTLIB::LOG10_FINITE_F64,
    4070             :                                         RTLIB::LOG10_FINITE_F80,
    4071             :                                         RTLIB::LOG10_FINITE_F128,
    4072             :                                         RTLIB::LOG10_FINITE_PPCF128));
    4073             :     else
    4074          70 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
    4075             :                                         RTLIB::LOG10_F80, RTLIB::LOG10_F128,
    4076             :                                         RTLIB::LOG10_PPCF128));
    4077             :     break;
    4078          86 :   case ISD::FEXP:
    4079             :   case ISD::STRICT_FEXP:
    4080          97 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
    4081           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
    4082             :                                         RTLIB::EXP_FINITE_F64,
    4083             :                                         RTLIB::EXP_FINITE_F80,
    4084             :                                         RTLIB::EXP_FINITE_F128,
    4085             :                                         RTLIB::EXP_FINITE_PPCF128));
    4086             :     else
    4087          82 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
    4088             :                                         RTLIB::EXP_F80, RTLIB::EXP_F128,
    4089             :                                         RTLIB::EXP_PPCF128));
    4090             :     break;
    4091          73 :   case ISD::FEXP2:
    4092             :   case ISD::STRICT_FEXP2:
    4093          84 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
    4094           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
    4095             :                                         RTLIB::EXP2_FINITE_F64,
    4096             :                                         RTLIB::EXP2_FINITE_F80,
    4097             :                                         RTLIB::EXP2_FINITE_F128,
    4098             :                                         RTLIB::EXP2_FINITE_PPCF128));
    4099             :     else
    4100          69 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
    4101             :                                         RTLIB::EXP2_F80, RTLIB::EXP2_F128,
    4102             :                                         RTLIB::EXP2_PPCF128));
    4103             :     break;
    4104          30 :   case ISD::FTRUNC:
    4105          30 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
    4106             :                                       RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
    4107             :                                       RTLIB::TRUNC_PPCF128));
    4108          30 :     break;
    4109          62 :   case ISD::FFLOOR:
    4110          62 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
    4111             :                                       RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
    4112             :                                       RTLIB::FLOOR_PPCF128));
    4113          62 :     break;
    4114          20 :   case ISD::FCEIL:
    4115          20 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
    4116             :                                       RTLIB::CEIL_F80, RTLIB::CEIL_F128,
    4117             :                                       RTLIB::CEIL_PPCF128));
    4118          20 :     break;
    4119          25 :   case ISD::FRINT:
    4120             :   case ISD::STRICT_FRINT:
    4121          25 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
    4122             :                                       RTLIB::RINT_F80, RTLIB::RINT_F128,
    4123             :                                       RTLIB::RINT_PPCF128));
    4124          25 :     break;
    4125          48 :   case ISD::FNEARBYINT:
    4126             :   case ISD::STRICT_FNEARBYINT:
    4127          48 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
    4128             :                                       RTLIB::NEARBYINT_F64,
    4129             :                                       RTLIB::NEARBYINT_F80,
    4130             :                                       RTLIB::NEARBYINT_F128,
    4131             :                                       RTLIB::NEARBYINT_PPCF128));
    4132          48 :     break;
    4133          17 :   case ISD::FROUND:
    4134          17 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
    4135             :                                       RTLIB::ROUND_F64,
    4136             :                                       RTLIB::ROUND_F80,
    4137             :                                       RTLIB::ROUND_F128,
    4138             :                                       RTLIB::ROUND_PPCF128));
    4139          17 :     break;
    4140         106 :   case ISD::FPOWI:
    4141             :   case ISD::STRICT_FPOWI:
    4142         106 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
    4143             :                                       RTLIB::POWI_F80, RTLIB::POWI_F128,
    4144             :                                       RTLIB::POWI_PPCF128));
    4145         106 :     break;
    4146         103 :   case ISD::FPOW:
    4147             :   case ISD::STRICT_FPOW:
    4148         114 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
    4149           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
    4150             :                                         RTLIB::POW_FINITE_F64,
    4151             :                                         RTLIB::POW_FINITE_F80,
    4152             :                                         RTLIB::POW_FINITE_F128,
    4153             :                                         RTLIB::POW_FINITE_PPCF128));
    4154             :     else
    4155          99 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
    4156             :                                         RTLIB::POW_F80, RTLIB::POW_F128,
    4157             :                                         RTLIB::POW_PPCF128));
    4158             :     break;
    4159           6 :   case ISD::FDIV:
    4160           6 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
    4161             :                                       RTLIB::DIV_F80, RTLIB::DIV_F128,
    4162             :                                       RTLIB::DIV_PPCF128));
    4163           6 :     break;
    4164          88 :   case ISD::FREM:
    4165          88 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
    4166             :                                       RTLIB::REM_F80, RTLIB::REM_F128,
    4167             :                                       RTLIB::REM_PPCF128));
    4168          88 :     break;
    4169         224 :   case ISD::FMA:
    4170             :   case ISD::STRICT_FMA:
    4171         224 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
    4172             :                                       RTLIB::FMA_F80, RTLIB::FMA_F128,
    4173             :                                       RTLIB::FMA_PPCF128));
    4174         224 :     break;
    4175          12 :   case ISD::FADD:
    4176          12 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
    4177             :                                       RTLIB::ADD_F80, RTLIB::ADD_F128,
    4178             :                                       RTLIB::ADD_PPCF128));
    4179          12 :     break;
    4180           7 :   case ISD::FMUL:
    4181           7 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
    4182             :                                       RTLIB::MUL_F80, RTLIB::MUL_F128,
    4183             :                                       RTLIB::MUL_PPCF128));
    4184           7 :     break;
    4185             :   case ISD::FP16_TO_FP:
    4186             :     if (Node->getValueType(0) == MVT::f32) {
    4187         345 :       Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
    4188             :     }
    4189             :     break;
    4190             :   case ISD::FP_TO_FP16: {
    4191             :     RTLIB::Libcall LC =
    4192         458 :         RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
    4193             :     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
    4194         458 :     Results.push_back(ExpandLibCall(LC, Node, false));
    4195         458 :     break;
    4196             :   }
    4197           4 :   case ISD::FSUB:
    4198           4 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
    4199             :                                       RTLIB::SUB_F80, RTLIB::SUB_F128,
    4200             :                                       RTLIB::SUB_PPCF128));
    4201           4 :     break;
    4202          19 :   case ISD::SREM:
    4203          19 :     Results.push_back(ExpandIntLibCall(Node, true,
    4204             :                                        RTLIB::SREM_I8,
    4205             :                                        RTLIB::SREM_I16, RTLIB::SREM_I32,
    4206             :                                        RTLIB::SREM_I64, RTLIB::SREM_I128));
    4207          19 :     break;
    4208          13 :   case ISD::UREM:
    4209          13 :     Results.push_back(ExpandIntLibCall(Node, false,
    4210             :                                        RTLIB::UREM_I8,
    4211             :                                        RTLIB::UREM_I16, RTLIB::UREM_I32,
    4212             :                                        RTLIB::UREM_I64, RTLIB::UREM_I128));
    4213          13 :     break;
    4214          48 :   case ISD::SDIV:
    4215          48 :     Results.push_back(ExpandIntLibCall(Node, true,
    4216             :                                        RTLIB::SDIV_I8,
    4217             :                                        RTLIB::SDIV_I16, RTLIB::SDIV_I32,
    4218             :                                        RTLIB::SDIV_I64, RTLIB::SDIV_I128));
    4219          48 :     break;
    4220          20 :   case ISD::UDIV:
    4221          20 :     Results.push_back(ExpandIntLibCall(Node, false,
    4222             :                                        RTLIB::UDIV_I8,
    4223             :                                        RTLIB::UDIV_I16, RTLIB::UDIV_I32,
    4224             :                                        RTLIB::UDIV_I64, RTLIB::UDIV_I128));
    4225          20 :     break;
    4226          14 :   case ISD::SDIVREM:
    4227             :   case ISD::UDIVREM:
    4228             :     // Expand into divrem libcall
    4229          14 :     ExpandDivRemLibCall(Node, Results);
    4230          14 :     break;
    4231          16 :   case ISD::MUL:
    4232          16 :     Results.push_back(ExpandIntLibCall(Node, false,
    4233             :                                        RTLIB::MUL_I8,
    4234             :                                        RTLIB::MUL_I16, RTLIB::MUL_I32,
    4235             :                                        RTLIB::MUL_I64, RTLIB::MUL_I128));
    4236          16 :     break;
    4237             :   }
    4238             : 
    4239             :   // Replace the original node with the legalized result.
    4240       36334 :   if (!Results.empty()) {
    4241             :     DEBUG(dbgs() << "Successfully converted node to libcall\n");
    4242        2724 :     ReplaceNode(Node, Results.data());
    4243             :   } else
    4244             :     DEBUG(dbgs() << "Could not convert node to libcall\n");
    4245       36334 : }
    4246             : 
    4247             : // Determine the vector type to use in place of an original scalar element when
    4248             : // promoting equally sized vectors.
    4249        2940 : static MVT getPromotedVectorElementType(const TargetLowering &TLI,
    4250             :                                         MVT EltVT, MVT NewEltVT) {
    4251        2940 :   unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
    4252        2940 :   MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
    4253             :   assert(TLI.isTypeLegal(MidVT) && "unexpected");
    4254        2940 :   return MidVT;
    4255             : }
    4256             : 
    4257        6493 : void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
    4258             :   DEBUG(dbgs() << "Trying to promote node\n");
    4259             :   SmallVector<SDValue, 8> Results;
    4260        6493 :   MVT OVT = Node->getSimpleValueType(0);
    4261        6444 :   if (Node->getOpcode() == ISD::UINT_TO_FP ||
    4262        6428 :       Node->getOpcode() == ISD::SINT_TO_FP ||
    4263        6231 :       Node->getOpcode() == ISD::SETCC ||
    4264       10195 :       Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
    4265             :       Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
    4266        2795 :     OVT = Node->getOperand(0).getSimpleValueType();
    4267             :   }
    4268        6493 :   if (Node->getOpcode() == ISD::BR_CC)
    4269           1 :     OVT = Node->getOperand(2).getSimpleValueType();
    4270        6493 :   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
    4271             :   SDLoc dl(Node);
    4272        6493 :   SDValue Tmp1, Tmp2, Tmp3;
    4273        6493 :   switch (Node->getOpcode()) {
    4274          39 :   case ISD::CTTZ:
    4275             :   case ISD::CTTZ_ZERO_UNDEF:
    4276             :   case ISD::CTLZ:
    4277             :   case ISD::CTLZ_ZERO_UNDEF:
    4278             :   case ISD::CTPOP:
    4279             :     // Zero extend the argument.
    4280          78 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
    4281          39 :     if (Node->getOpcode() == ISD::CTTZ) {
    4282             :       // The count is the same in the promoted type except if the original
    4283             :       // value was zero.  This can be handled by setting the bit just off
    4284             :       // the top of the original type.
    4285             :       auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
    4286           5 :                                         OVT.getSizeInBits());
    4287          10 :       Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
    4288          10 :                          DAG.getConstant(TopBit, dl, NVT));
    4289             :     }
    4290             :     // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
    4291             :     // already the correct result.
    4292          78 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4293          39 :     if (Node->getOpcode() == ISD::CTLZ ||
    4294             :         Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
    4295             :       // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
    4296          22 :       Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
    4297          22 :                           DAG.getConstant(NVT.getSizeInBits() -
    4298          33 :                                           OVT.getSizeInBits(), dl, NVT));
    4299             :     }
    4300          78 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4301          39 :     break;
    4302           3 :   case ISD::BITREVERSE:
    4303             :   case ISD::BSWAP: {
    4304           3 :     unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
    4305           6 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
    4306           6 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4307           6 :     Tmp1 = DAG.getNode(
    4308             :         ISD::SRL, dl, NVT, Tmp1,
    4309             :         DAG.getConstant(DiffBits, dl,
    4310           9 :                         TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
    4311             : 
    4312           6 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4313           3 :     break;
    4314             :   }
    4315         108 :   case ISD::FP_TO_UINT:
    4316             :   case ISD::FP_TO_SINT:
    4317         108 :     Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
    4318         108 :                                  Node->getOpcode() == ISD::FP_TO_SINT, dl);
    4319         108 :     Results.push_back(Tmp1);
    4320         108 :     break;
    4321          65 :   case ISD::UINT_TO_FP:
    4322             :   case ISD::SINT_TO_FP:
    4323          65 :     Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
    4324          65 :                                  Node->getOpcode() == ISD::SINT_TO_FP, dl);
    4325          65 :     Results.push_back(Tmp1);
    4326          65 :     break;
    4327           1 :   case ISD::VAARG: {
    4328           1 :     SDValue Chain = Node->getOperand(0); // Get the chain.
    4329           1 :     SDValue Ptr = Node->getOperand(1); // Get the pointer.
    4330             : 
    4331             :     unsigned TruncOp;
    4332           1 :     if (OVT.isVector()) {
    4333             :       TruncOp = ISD::BITCAST;
    4334             :     } else {
    4335             :       assert(OVT.isInteger()
    4336             :         && "VAARG promotion is supported only for vectors or integer types");
    4337             :       TruncOp = ISD::TRUNCATE;
    4338             :     }
    4339             : 
    4340             :     // Perform the larger operation, then convert back
    4341           2 :     Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
    4342           2 :              Node->getConstantOperandVal(3));
    4343           1 :     Chain = Tmp1.getValue(1);
    4344             : 
    4345           2 :     Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
    4346             : 
    4347             :     // Modified the chain result - switch anything that used the old chain to
    4348             :     // use the new one.
    4349           2 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
    4350           2 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
    4351           1 :     if (UpdatedNodes) {
    4352           0 :       UpdatedNodes->insert(Tmp2.getNode());
    4353           0 :       UpdatedNodes->insert(Chain.getNode());
    4354             :     }
    4355           1 :     ReplacedNode(Node);
    4356             :     break;
    4357             :   }
    4358             :   case ISD::MUL:
    4359             :   case ISD::SDIV:
    4360             :   case ISD::SREM:
    4361             :   case ISD::UDIV:
    4362             :   case ISD::UREM:
    4363             :   case ISD::AND:
    4364             :   case ISD::OR:
    4365             :   case ISD::XOR: {
    4366             :     unsigned ExtOp, TruncOp;
    4367        1933 :     if (OVT.isVector()) {
    4368             :       ExtOp   = ISD::BITCAST;
    4369             :       TruncOp = ISD::BITCAST;
    4370             :     } else {
    4371             :       assert(OVT.isInteger() && "Cannot promote logic operation");
    4372             : 
    4373          11 :       switch (Node->getOpcode()) {
    4374             :       default:
    4375             :         ExtOp = ISD::ANY_EXTEND;
    4376             :         break;
    4377           4 :       case ISD::SDIV:
    4378             :       case ISD::SREM:
    4379             :         ExtOp = ISD::SIGN_EXTEND;
    4380           4 :         break;
    4381           6 :       case ISD::UDIV:
    4382             :       case ISD::UREM:
    4383             :         ExtOp = ISD::ZERO_EXTEND;
    4384           6 :         break;
    4385             :       }
    4386             :       TruncOp = ISD::TRUNCATE;
    4387             :     }
    4388             :     // Promote each of the values to the new type.
    4389        3866 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4390        3866 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4391             :     // Perform the larger operation, then convert back
    4392        3866 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
    4393        3866 :     Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
    4394        1933 :     break;
    4395             :   }
    4396           0 :   case ISD::UMUL_LOHI:
    4397             :   case ISD::SMUL_LOHI: {
    4398             :     // Promote to a multiply in a wider integer type.
    4399           0 :     unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
    4400             :                                                          : ISD::SIGN_EXTEND;
    4401           0 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4402           0 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4403           0 :     Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
    4404             : 
    4405           0 :     auto &DL = DAG.getDataLayout();
    4406           0 :     unsigned OriginalSize = OVT.getScalarSizeInBits();
    4407           0 :     Tmp2 = DAG.getNode(
    4408             :         ISD::SRL, dl, NVT, Tmp1,
    4409           0 :         DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
    4410           0 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4411           0 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
    4412           0 :     break;
    4413             :   }
    4414         164 :   case ISD::SELECT: {
    4415             :     unsigned ExtOp, TruncOp;
    4416         656 :     if (Node->getValueType(0).isVector() ||
    4417         328 :         Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
    4418             :       ExtOp   = ISD::BITCAST;
    4419             :       TruncOp = ISD::BITCAST;
    4420           8 :     } else if (Node->getValueType(0).isInteger()) {
    4421             :       ExtOp   = ISD::ANY_EXTEND;
    4422             :       TruncOp = ISD::TRUNCATE;
    4423             :     } else {
    4424             :       ExtOp   = ISD::FP_EXTEND;
    4425             :       TruncOp = ISD::FP_ROUND;
    4426             :     }
    4427         164 :     Tmp1 = Node->getOperand(0);
    4428             :     // Promote each of the values to the new type.
    4429         328 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4430         328 :     Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
    4431             :     // Perform the larger operation, then round down.
    4432         328 :     Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
    4433         164 :     if (TruncOp != ISD::FP_ROUND)
    4434         320 :       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
    4435             :     else
    4436           8 :       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
    4437           8 :                          DAG.getIntPtrConstant(0, dl));
    4438         164 :     Results.push_back(Tmp1);
    4439         164 :     break;
    4440             :   }
    4441             :   case ISD::VECTOR_SHUFFLE: {
    4442         507 :     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
    4443             : 
    4444             :     // Cast the two input vectors.
    4445        1014 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
    4446        1014 :     Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
    4447             : 
    4448             :     // Convert the shuffle mask to the right # elements.
    4449         507 :     Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
    4450        1014 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
    4451         507 :     Results.push_back(Tmp1);
    4452             :     break;
    4453             :   }
    4454         197 :   case ISD::SETCC: {
    4455             :     unsigned ExtOp = ISD::FP_EXTEND;
    4456         197 :     if (NVT.isInteger()) {
    4457             :       ISD::CondCode CCCode =
    4458             :         cast<CondCodeSDNode>(Node->getOperand(2))->get();
    4459          80 :       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    4460             :     }
    4461         394 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4462         394 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4463         394 :     Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
    4464             :                                   Tmp1, Tmp2, Node->getOperand(2)));
    4465         197 :     break;
    4466             :   }
    4467           1 :   case ISD::BR_CC: {
    4468             :     unsigned ExtOp = ISD::FP_EXTEND;
    4469           1 :     if (NVT.isInteger()) {
    4470             :       ISD::CondCode CCCode =
    4471             :         cast<CondCodeSDNode>(Node->getOperand(1))->get();
    4472           0 :       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    4473             :     }
    4474           2 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
    4475           2 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
    4476           2 :     Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
    4477             :                                   Node->getOperand(0), Node->getOperand(1),
    4478             :                                   Tmp1, Tmp2, Node->getOperand(4)));
    4479           1 :     break;
    4480             :   }
    4481         183 :   case ISD::FADD:
    4482             :   case ISD::FSUB:
    4483             :   case ISD::FMUL:
    4484             :   case ISD::FDIV:
    4485             :   case ISD::FREM:
    4486             :   case ISD::FMINNUM:
    4487             :   case ISD::FMAXNUM:
    4488             :   case ISD::FPOW:
    4489         366 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4490         366 :     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
    4491         366 :     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
    4492         183 :                        Node->getFlags());
    4493         366 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4494         366 :                                   Tmp3, DAG.getIntPtrConstant(0, dl)));
    4495         183 :     break;
    4496          25 :   case ISD::FMA:
    4497          50 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4498          50 :     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
    4499          50 :     Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
    4500          25 :     Results.push_back(
    4501          75 :         DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4502          25 :                     DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
    4503          75 :                     DAG.getIntPtrConstant(0, dl)));
    4504          25 :     break;
    4505          12 :   case ISD::FCOPYSIGN:
    4506             :   case ISD::FPOWI: {
    4507          24 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4508          12 :     Tmp2 = Node->getOperand(1);
    4509          24 :     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
    4510             : 
    4511             :     // fcopysign doesn't change anything but the sign bit, so
    4512             :     //   (fp_round (fcopysign (fpext a), b))
    4513             :     // is as precise as
    4514             :     //   (fp_round (fpext a))
    4515             :     // which is a no-op. Mark it as a TRUNCating FP_ROUND.
    4516          12 :     const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
    4517          24 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4518          24 :                                   Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
    4519          12 :     break;
    4520             :   }
    4521         163 :   case ISD::FFLOOR:
    4522             :   case ISD::FCEIL:
    4523             :   case ISD::FRINT:
    4524             :   case ISD::FNEARBYINT:
    4525             :   case ISD::FROUND:
    4526             :   case ISD::FTRUNC:
    4527             :   case ISD::FNEG:
    4528             :   case ISD::FSQRT:
    4529             :   case ISD::FSIN:
    4530             :   case ISD::FCOS:
    4531             :   case ISD::FLOG:
    4532             :   case ISD::FLOG2:
    4533             :   case ISD::FLOG10:
    4534             :   case ISD::FABS:
    4535             :   case ISD::FEXP:
    4536             :   case ISD::FEXP2:
    4537         326 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4538         326 :     Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4539         326 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4540         326 :                                   Tmp2, DAG.getIntPtrConstant(0, dl)));
    4541         163 :     break;
    4542         376 :   case ISD::BUILD_VECTOR: {
    4543         376 :     MVT EltVT = OVT.getVectorElementType();
    4544         376 :     MVT NewEltVT = NVT.getVectorElementType();
    4545             : 
    4546             :     // Handle bitcasts to a different vector type with the same total bit size
    4547             :     //
    4548             :     // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
    4549             :     //  =>
    4550             :     //  v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
    4551             : 
    4552             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4553             :            "Invalid promote type for build_vector");
    4554             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4555             : 
    4556         376 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4557             : 
    4558             :     SmallVector<SDValue, 8> NewOps;
    4559        1880 :     for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
    4560         752 :       SDValue Op = Node->getOperand(I);
    4561        2256 :       NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
    4562             :     }
    4563             : 
    4564             :     SDLoc SL(Node);
    4565         752 :     SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
    4566         752 :     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
    4567         376 :     Results.push_back(CvtVec);
    4568             :     break;
    4569             :   }
    4570        2529 :   case ISD::EXTRACT_VECTOR_ELT: {
    4571        2529 :     MVT EltVT = OVT.getVectorElementType();
    4572        2529 :     MVT NewEltVT = NVT.getVectorElementType();
    4573             : 
    4574             :     // Handle bitcasts to a different vector type with the same total bit size.
    4575             :     //
    4576             :     // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
    4577             :     //  =>
    4578             :     //  v4i32:castx = bitcast x:v2i64
    4579             :     //
    4580             :     // i64 = bitcast
    4581             :     //   (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
    4582             :     //                       (i32 (extract_vector_elt castx, (2 * y + 1)))
    4583             :     //
    4584             : 
    4585             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4586             :            "Invalid promote type for extract_vector_elt");
    4587             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4588             : 
    4589        2529 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4590        2529 :     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
    4591             : 
    4592        2529 :     SDValue Idx = Node->getOperand(1);
    4593        2529 :     EVT IdxVT = Idx.getValueType();
    4594             :     SDLoc SL(Node);
    4595        2529 :     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
    4596        5058 :     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
    4597             : 
    4598        5058 :     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
    4599             : 
    4600             :     SmallVector<SDValue, 8> NewOps;
    4601       12645 :     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
    4602        5058 :       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
    4603       10116 :       SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
    4604             : 
    4605        5058 :       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
    4606        5058 :                                 CastVec, TmpIdx);
    4607        5058 :       NewOps.push_back(Elt);
    4608             :     }
    4609             : 
    4610        5058 :     SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
    4611        5058 :     Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
    4612             :     break;
    4613             :   }
    4614           4 :   case ISD::INSERT_VECTOR_ELT: {
    4615           4 :     MVT EltVT = OVT.getVectorElementType();
    4616           4 :     MVT NewEltVT = NVT.getVectorElementType();
    4617             : 
    4618             :     // Handle bitcasts to a different vector type with the same total bit size
    4619             :     //
    4620             :     // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
    4621             :     //  =>
    4622             :     //  v4i32:castx = bitcast x:v2i64
    4623             :     //  v2i32:casty = bitcast y:i64
    4624             :     //
    4625             :     // v2i64 = bitcast
    4626             :     //   (v4i32 insert_vector_elt
    4627             :     //       (v4i32 insert_vector_elt v4i32:castx,
    4628             :     //                                (extract_vector_elt casty, 0), 2 * z),
    4629             :     //        (extract_vector_elt casty, 1), (2 * z + 1))
    4630             : 
    4631             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4632             :            "Invalid promote type for insert_vector_elt");
    4633             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4634             : 
    4635           4 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4636           4 :     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
    4637             : 
    4638           4 :     SDValue Val = Node->getOperand(1);
    4639           4 :     SDValue Idx = Node->getOperand(2);
    4640           4 :     EVT IdxVT = Idx.getValueType();
    4641             :     SDLoc SL(Node);
    4642             : 
    4643           8 :     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
    4644           8 :     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
    4645             : 
    4646           8 :     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
    4647           8 :     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
    4648             : 
    4649           4 :     SDValue NewVec = CastVec;
    4650          20 :     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
    4651           8 :       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
    4652          16 :       SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
    4653             : 
    4654           8 :       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
    4655           8 :                                 CastVal, IdxOffset);
    4656             : 
    4657          16 :       NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
    4658           8 :                            NewVec, Elt, InEltIdx);
    4659             :     }
    4660             : 
    4661           8 :     Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
    4662             :     break;
    4663             :   }
    4664          31 :   case ISD::SCALAR_TO_VECTOR: {
    4665          31 :     MVT EltVT = OVT.getVectorElementType();
    4666          31 :     MVT NewEltVT = NVT.getVectorElementType();
    4667             : 
    4668             :     // Handle bitcasts to different vector type with the same total bit size.
    4669             :     //
    4670             :     // e.g. v2i64 = scalar_to_vector x:i64
    4671             :     //   =>
    4672             :     //  concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
    4673             :     //
    4674             : 
    4675          31 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4676          31 :     SDValue Val = Node->getOperand(0);
    4677             :     SDLoc SL(Node);
    4678             : 
    4679          62 :     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
    4680          62 :     SDValue Undef = DAG.getUNDEF(MidVT);
    4681             : 
    4682             :     SmallVector<SDValue, 8> NewElts;
    4683          31 :     NewElts.push_back(CastVal);
    4684          62 :     for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
    4685          31 :       NewElts.push_back(Undef);
    4686             : 
    4687          62 :     SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
    4688          62 :     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
    4689          31 :     Results.push_back(CvtVec);
    4690             :     break;
    4691             :   }
    4692             :   }
    4693             : 
    4694             :   // Replace the original node with the legalized result.
    4695        6493 :   if (!Results.empty()) {
    4696             :     DEBUG(dbgs() << "Successfully promoted node\n");
    4697        6340 :     ReplaceNode(Node, Results.data());
    4698             :   } else
    4699             :     DEBUG(dbgs() << "Could not promote node\n");
    4700        6493 : }
    4701             : 
    4702             : /// This is the entry point for the file.
    4703      332248 : void SelectionDAG::Legalize() {
    4704      332248 :   AssignTopologicalOrder();
    4705             : 
    4706             :   SmallPtrSet<SDNode *, 16> LegalizedNodes;
    4707             :   // Use a delete listener to remove nodes which were deleted during
    4708             :   // legalization from LegalizeNodes. This is needed to handle the situation
    4709             :   // where a new node is allocated by the object pool to the same address of a
    4710             :   // previously deleted node.
    4711             :   DAGNodeDeletedListener DeleteListener(
    4712             :       *this,
    4713      332501 :       [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
    4714             : 
    4715             :   SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
    4716             : 
    4717             :   // Visit all the nodes. We start in topological order, so that we see
    4718             :   // nodes with their original operands intact. Legalization can produce
    4719             :   // new nodes which may themselves need to be legalized. Iterate until all
    4720             :   // nodes have been legalized.
    4721             :   while (true) {
    4722             :     bool AnyLegalized = false;
    4723    31591649 :     for (auto NI = allnodes_end(); NI != allnodes_begin();) {
    4724             :       --NI;
    4725             : 
    4726             :       SDNode *N = &*NI;
    4727    31146296 :       if (N->use_empty() && N != getRoot().getNode()) {
    4728             :         ++NI;
    4729      491099 :         DeleteNode(N);
    4730      491099 :         continue;
    4731             :       }
    4732             : 
    4733    30164098 :       if (LegalizedNodes.insert(N).second) {
    4734             :         AnyLegalized = true;
    4735    11605341 :         Legalizer.LegalizeOp(N);
    4736             : 
    4737    11605340 :         if (N->use_empty() && N != getRoot().getNode()) {
    4738             :           ++NI;
    4739     1105904 :           DeleteNode(N);
    4740             :         }
    4741             :       }
    4742             :     }
    4743      936452 :     if (!AnyLegalized)
    4744             :       break;
    4745             : 
    4746             :   }
    4747             : 
    4748             :   // Remove dead nodes now.
    4749      332247 :   RemoveDeadNodes();
    4750      332247 : }
    4751             : 
    4752    11432231 : bool SelectionDAG::LegalizeOp(SDNode *N,
    4753             :                               SmallSetVector<SDNode *, 16> &UpdatedNodes) {
    4754             :   SmallPtrSet<SDNode *, 16> LegalizedNodes;
    4755             :   SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
    4756             : 
    4757             :   // Directly insert the node in question, and legalize it. This will recurse
    4758             :   // as needed through operands.
    4759    11432231 :   LegalizedNodes.insert(N);
    4760    11432231 :   Legalizer.LegalizeOp(N);
    4761             : 
    4762    22864462 :   return LegalizedNodes.count(N);
    4763             : }

Generated by: LCOV version 1.13