LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeDAG.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 2020 2260 89.4 %
Date: 2018-06-17 00:07:59 Functions: 44 46 95.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the SelectionDAG::Legalize method.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/ADT/APFloat.h"
      15             : #include "llvm/ADT/APInt.h"
      16             : #include "llvm/ADT/ArrayRef.h"
      17             : #include "llvm/ADT/SetVector.h"
      18             : #include "llvm/ADT/SmallPtrSet.h"
      19             : #include "llvm/ADT/SmallSet.h"
      20             : #include "llvm/ADT/SmallVector.h"
      21             : #include "llvm/CodeGen/ISDOpcodes.h"
      22             : #include "llvm/CodeGen/MachineFunction.h"
      23             : #include "llvm/CodeGen/MachineJumpTableInfo.h"
      24             : #include "llvm/CodeGen/MachineMemOperand.h"
      25             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      26             : #include "llvm/CodeGen/SelectionDAG.h"
      27             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      28             : #include "llvm/CodeGen/TargetFrameLowering.h"
      29             : #include "llvm/CodeGen/TargetLowering.h"
      30             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      31             : #include "llvm/CodeGen/ValueTypes.h"
      32             : #include "llvm/IR/CallingConv.h"
      33             : #include "llvm/IR/Constants.h"
      34             : #include "llvm/IR/DataLayout.h"
      35             : #include "llvm/IR/DerivedTypes.h"
      36             : #include "llvm/IR/Function.h"
      37             : #include "llvm/IR/Metadata.h"
      38             : #include "llvm/IR/Type.h"
      39             : #include "llvm/Support/Casting.h"
      40             : #include "llvm/Support/Compiler.h"
      41             : #include "llvm/Support/Debug.h"
      42             : #include "llvm/Support/ErrorHandling.h"
      43             : #include "llvm/Support/MachineValueType.h"
      44             : #include "llvm/Support/MathExtras.h"
      45             : #include "llvm/Support/raw_ostream.h"
      46             : #include "llvm/Target/TargetMachine.h"
      47             : #include "llvm/Target/TargetOptions.h"
      48             : #include <algorithm>
      49             : #include <cassert>
      50             : #include <cstdint>
      51             : #include <tuple>
      52             : #include <utility>
      53             : 
      54             : using namespace llvm;
      55             : 
      56             : #define DEBUG_TYPE "legalizedag"
      57             : 
      58             : namespace {
      59             : 
      60             : /// Keeps track of state when getting the sign of a floating-point value as an
      61             : /// integer.
      62          79 : struct FloatSignAsInt {
      63             :   EVT FloatVT;
      64             :   SDValue Chain;
      65             :   SDValue FloatPtr;
      66             :   SDValue IntPtr;
      67             :   MachinePointerInfo IntPointerInfo;
      68             :   MachinePointerInfo FloatPointerInfo;
      69             :   SDValue IntValue;
      70             :   APInt SignMask;
      71             :   uint8_t SignBit;
      72             : };
      73             : 
      74             : //===----------------------------------------------------------------------===//
      75             : /// This takes an arbitrary SelectionDAG as input and
      76             : /// hacks on it until the target machine can handle it.  This involves
      77             : /// eliminating value sizes the machine cannot handle (promoting small sizes to
      78             : /// large sizes or splitting up large values into small values) as well as
      79             : /// eliminating operations the machine cannot handle.
      80             : ///
      81             : /// This code also does a small amount of optimization and recognition of idioms
      82             : /// as part of its processing.  For example, if a target does not support a
      83             : /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
      84             : /// will attempt merge setcc and brc instructions into brcc's.
      85             : class SelectionDAGLegalize {
      86             :   const TargetMachine &TM;
      87             :   const TargetLowering &TLI;
      88             :   SelectionDAG &DAG;
      89             : 
      90             :   /// The set of nodes which have already been legalized. We hold a
      91             :   /// reference to it in order to update as necessary on node deletion.
      92             :   SmallPtrSetImpl<SDNode *> &LegalizedNodes;
      93             : 
      94             :   /// A set of all the nodes updated during legalization.
      95             :   SmallSetVector<SDNode *, 16> *UpdatedNodes;
      96             : 
      97         461 :   EVT getSetCCResultType(EVT VT) const {
      98         922 :     return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
      99             :   }
     100             : 
     101             :   // Libcall insertion helpers.
     102             : 
     103             : public:
     104             :   SelectionDAGLegalize(SelectionDAG &DAG,
     105             :                        SmallPtrSetImpl<SDNode *> &LegalizedNodes,
     106             :                        SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
     107    24076990 :       : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
     108    24076990 :         LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
     109             : 
     110             :   /// Legalizes the given operation.
     111             :   void LegalizeOp(SDNode *Node);
     112             : 
     113             : private:
     114             :   SDValue OptimizeFloatStore(StoreSDNode *ST);
     115             : 
     116             :   void LegalizeLoadOps(SDNode *Node);
     117             :   void LegalizeStoreOps(SDNode *Node);
     118             : 
     119             :   /// Some targets cannot handle a variable
     120             :   /// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
     121             :   /// is necessary to spill the vector being inserted into to memory, perform
     122             :   /// the insert there, and then read the result back.
     123             :   SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
     124             :                                          const SDLoc &dl);
     125             :   SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
     126             :                                   const SDLoc &dl);
     127             : 
     128             :   /// Return a vector shuffle operation which
     129             :   /// performs the same shuffe in terms of order or result bytes, but on a type
     130             :   /// whose vector element type is narrower than the original shuffle type.
     131             :   /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
     132             :   SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
     133             :                                      SDValue N1, SDValue N2,
     134             :                                      ArrayRef<int> Mask) const;
     135             : 
     136             :   bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
     137             :                              bool &NeedInvert, const SDLoc &dl);
     138             : 
     139             :   SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
     140             :   SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
     141             :                         unsigned NumOps, bool isSigned, const SDLoc &dl);
     142             : 
     143             :   std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
     144             :                                                  SDNode *Node, bool isSigned);
     145             :   SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
     146             :                           RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
     147             :                           RTLIB::Libcall Call_F128,
     148             :                           RTLIB::Libcall Call_PPCF128);
     149             :   SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
     150             :                            RTLIB::Libcall Call_I8,
     151             :                            RTLIB::Libcall Call_I16,
     152             :                            RTLIB::Libcall Call_I32,
     153             :                            RTLIB::Libcall Call_I64,
     154             :                            RTLIB::Libcall Call_I128);
     155             :   void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
     156             :   void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
     157             : 
     158             :   SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
     159             :                            const SDLoc &dl);
     160             :   SDValue ExpandBUILD_VECTOR(SDNode *Node);
     161             :   SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
     162             :   void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
     163             :                                 SmallVectorImpl<SDValue> &Results);
     164             :   void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
     165             :                          SDValue Value) const;
     166             :   SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
     167             :                           SDValue NewIntValue) const;
     168             :   SDValue ExpandFCOPYSIGN(SDNode *Node) const;
     169             :   SDValue ExpandFABS(SDNode *Node) const;
     170             :   SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
     171             :                                const SDLoc &dl);
     172             :   SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
     173             :                                 const SDLoc &dl);
     174             :   SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
     175             :                                 const SDLoc &dl);
     176             : 
     177             :   SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
     178             :   SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
     179             :   SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
     180             : 
     181             :   SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
     182             :   SDValue ExpandInsertToVectorThroughStack(SDValue Op);
     183             :   SDValue ExpandVectorBuildThroughStack(SDNode* Node);
     184             : 
     185             :   SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
     186             :   SDValue ExpandConstant(ConstantSDNode *CP);
     187             : 
     188             :   // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
     189             :   bool ExpandNode(SDNode *Node);
     190             :   void ConvertNodeToLibcall(SDNode *Node);
     191             :   void PromoteNode(SDNode *Node);
     192             : 
     193             : public:
     194             :   // Node replacement helpers
     195             : 
     196     1143864 :   void ReplacedNode(SDNode *N) {
     197     1143864 :     LegalizedNodes.erase(N);
     198     1143864 :     if (UpdatedNodes)
     199       10297 :       UpdatedNodes->insert(N);
     200     1143864 :   }
     201             : 
     202         142 :   void ReplaceNode(SDNode *Old, SDNode *New) {
     203             :     LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
     204             :                dbgs() << "     with:      "; New->dump(&DAG));
     205             : 
     206             :     assert(Old->getNumValues() == New->getNumValues() &&
     207             :            "Replacing one node with another that produces a different number "
     208             :            "of values!");
     209         142 :     DAG.ReplaceAllUsesWith(Old, New);
     210         142 :     if (UpdatedNodes)
     211           1 :       UpdatedNodes->insert(New);
     212         142 :     ReplacedNode(Old);
     213         142 :   }
     214             : 
     215      710351 :   void ReplaceNode(SDValue Old, SDValue New) {
     216             :     LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
     217             :                dbgs() << "     with:      "; New->dump(&DAG));
     218             : 
     219      710351 :     DAG.ReplaceAllUsesWith(Old, New);
     220      710351 :     if (UpdatedNodes)
     221        3923 :       UpdatedNodes->insert(New.getNode());
     222      710351 :     ReplacedNode(Old.getNode());
     223      710351 :   }
     224             : 
     225      351080 :   void ReplaceNode(SDNode *Old, const SDValue *New) {
     226             :     LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
     227             : 
     228      351080 :     DAG.ReplaceAllUsesWith(Old, New);
     229     2008740 :     for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
     230             :       LLVM_DEBUG(dbgs() << (i == 0 ? "     with:      " : "      and:      ");
     231             :                  New[i]->dump(&DAG));
     232      653290 :       if (UpdatedNodes)
     233        4440 :         UpdatedNodes->insert(New[i].getNode());
     234             :     }
     235      351080 :     ReplacedNode(Old);
     236      351080 :   }
     237             : };
     238             : 
     239             : } // end anonymous namespace
     240             : 
     241             : /// Return a vector shuffle operation which
     242             : /// performs the same shuffe in terms of order or result bytes, but on a type
     243             : /// whose vector element type is narrower than the original shuffle type.
     244             : /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
     245         509 : SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
     246             :     EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
     247             :     ArrayRef<int> Mask) const {
     248         509 :   unsigned NumMaskElts = VT.getVectorNumElements();
     249         509 :   unsigned NumDestElts = NVT.getVectorNumElements();
     250         509 :   unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
     251             : 
     252             :   assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
     253             : 
     254         509 :   if (NumEltsGrowth == 1)
     255           0 :     return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
     256             : 
     257             :   SmallVector<int, 8> NewMask;
     258        6413 :   for (unsigned i = 0; i != NumMaskElts; ++i) {
     259        5904 :     int Idx = Mask[i];
     260       20104 :     for (unsigned j = 0; j != NumEltsGrowth; ++j) {
     261        8576 :       if (Idx < 0)
     262         920 :         NewMask.push_back(-1);
     263             :       else
     264        7656 :         NewMask.push_back(Idx * NumEltsGrowth + j);
     265             :     }
     266             :   }
     267             :   assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
     268             :   assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
     269        1018 :   return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
     270             : }
     271             : 
     272             : /// Expands the ConstantFP node to an integer constant or
     273             : /// a load from the constant pool.
     274             : SDValue
     275        3606 : SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
     276             :   bool Extend = false;
     277             :   SDLoc dl(CFP);
     278             : 
     279             :   // If a FP immediate is precise when represented as a float and if the
     280             :   // target can do an extending load from float to double, we put it into
     281             :   // the constant pool as a float, even if it's is statically typed as a
     282             :   // double.  This shrinks FP constants and canonicalizes them for targets where
     283             :   // an FP extending load is the same cost as a normal load (such as on the x87
     284             :   // fp stack or PPC FP unit).
     285        7212 :   EVT VT = CFP->getValueType(0);
     286        3606 :   ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
     287        3606 :   if (!UseCP) {
     288             :     assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
     289           0 :     return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
     290           0 :                            (VT == MVT::f64) ? MVT::i64 : MVT::i32);
     291             :   }
     292             : 
     293             :   APFloat APF = CFP->getValueAPF();
     294        3606 :   EVT OrigVT = VT;
     295        3606 :   EVT SVT = VT;
     296             : 
     297             :   // We don't want to shrink SNaNs. Converting the SNaN back to its real type
     298             :   // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
     299        3606 :   if (!APF.isSignaling()) {
     300        5394 :     while (SVT != MVT::f32 && SVT != MVT::f16) {
     301        1790 :       SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
     302        3158 :       if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
     303             :           // Only do this if the target has a native EXTLOAD instruction from
     304             :           // smaller type.
     305        2863 :           TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
     306        1073 :           TLI.ShouldShrinkFPConstant(OrigVT)) {
     307         538 :         Type *SType = SVT.getTypeForEVT(*DAG.getContext());
     308         538 :         LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
     309         538 :         VT = SVT;
     310             :         Extend = true;
     311             :       }
     312             :     }
     313             :   }
     314             : 
     315             :   SDValue CPIdx =
     316        7212 :       DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
     317        3606 :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
     318        3606 :   if (Extend) {
     319         445 :     SDValue Result = DAG.getExtLoad(
     320         445 :         ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
     321             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
     322         890 :         Alignment);
     323         445 :     return Result;
     324             :   }
     325        3161 :   SDValue Result = DAG.getLoad(
     326        3161 :       OrigVT, dl, DAG.getEntryNode(), CPIdx,
     327        6322 :       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
     328        3161 :   return Result;
     329             : }
     330             : 
     331             : /// Expands the Constant node to a load from the constant pool.
     332           0 : SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
     333             :   SDLoc dl(CP);
     334           0 :   EVT VT = CP->getValueType(0);
     335           0 :   SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
     336           0 :                                       TLI.getPointerTy(DAG.getDataLayout()));
     337           0 :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
     338           0 :   SDValue Result = DAG.getLoad(
     339           0 :       VT, dl, DAG.getEntryNode(), CPIdx,
     340           0 :       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
     341           0 :   return Result;
     342             : }
     343             : 
     344             : /// Some target cannot handle a variable insertion index for the
     345             : /// INSERT_VECTOR_ELT instruction.  In this case, it
     346             : /// is necessary to spill the vector being inserted into to memory, perform
     347             : /// the insert there, and then read the result back.
     348          22 : SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
     349             :                                                              SDValue Val,
     350             :                                                              SDValue Idx,
     351             :                                                              const SDLoc &dl) {
     352          22 :   SDValue Tmp1 = Vec;
     353          22 :   SDValue Tmp2 = Val;
     354          22 :   SDValue Tmp3 = Idx;
     355             : 
     356             :   // If the target doesn't support this, we have to spill the input vector
     357             :   // to a temporary stack slot, update the element, then reload it.  This is
     358             :   // badness.  We could also load the value into a vector register (either
     359             :   // with a "move to register" or "extload into register" instruction, then
     360             :   // permute it into place, if the idx is a constant and if the idx is
     361             :   // supported by the target.
     362          22 :   EVT VT    = Tmp1.getValueType();
     363          22 :   EVT EltVT = VT.getVectorElementType();
     364          22 :   SDValue StackPtr = DAG.CreateStackTemporary(VT);
     365             : 
     366          22 :   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
     367             : 
     368             :   // Store the vector.
     369          22 :   SDValue Ch = DAG.getStore(
     370          22 :       DAG.getEntryNode(), dl, Tmp1, StackPtr,
     371          44 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
     372             : 
     373          22 :   SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
     374             : 
     375             :   // Store the scalar value.
     376          44 :   Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
     377             :   // Load the updated vector.
     378          22 :   return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
     379          22 :                                                DAG.getMachineFunction(), SPFI));
     380             : }
     381             : 
     382        1060 : SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
     383             :                                                       SDValue Idx,
     384             :                                                       const SDLoc &dl) {
     385             :   if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
     386             :     // SCALAR_TO_VECTOR requires that the type of the value being inserted
     387             :     // match the element type of the vector being created, except for
     388             :     // integers in which case the inserted value can be over width.
     389        1038 :     EVT EltVT = Vec.getValueType().getVectorElementType();
     390        1039 :     if (Val.getValueType() == EltVT ||
     391           2 :         (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
     392        1038 :       SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
     393        1038 :                                   Vec.getValueType(), Val);
     394             : 
     395        1038 :       unsigned NumElts = Vec.getValueType().getVectorNumElements();
     396             :       // We generate a shuffle of InVec and ScVec, so the shuffle mask
     397             :       // should be 0,1,2,3,4,5... with the appropriate element replaced with
     398             :       // elt 0 of the RHS.
     399             :       SmallVector<int, 8> ShufOps;
     400        7750 :       for (unsigned i = 0; i != NumElts; ++i)
     401        6712 :         ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
     402             : 
     403        2076 :       return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
     404             :     }
     405             :   }
     406          22 :   return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
     407             : }
     408             : 
     409     1389387 : SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
     410             :   LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
     411             :   // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
     412             :   // FIXME: We shouldn't do this for TargetConstantFP's.
     413             :   // FIXME: move this to the DAG Combiner!  Note that we can't regress due
     414             :   // to phase ordering between legalized code and the dag combiner.  This
     415             :   // probably means that we need to integrate dag combiner and legalizer
     416             :   // together.
     417             :   // We generally can't do this one for long doubles.
     418     1389387 :   SDValue Chain = ST->getChain();
     419     1389387 :   SDValue Ptr = ST->getBasePtr();
     420     1389387 :   unsigned Alignment = ST->getAlignment();
     421     1389387 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
     422             :   AAMDNodes AAInfo = ST->getAAInfo();
     423             :   SDLoc dl(ST);
     424             :   if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
     425         224 :     if (CFP->getValueType(0) == MVT::f32 &&
     426           1 :         TLI.isTypeLegal(MVT::i32)) {
     427           2 :       SDValue Con = DAG.getConstant(CFP->getValueAPF().
     428           3 :                                       bitcastToAPInt().zextOrTrunc(32),
     429           2 :                                     SDLoc(CFP), MVT::i32);
     430           2 :       return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
     431           2 :                           MMOFlags, AAInfo);
     432             :     }
     433             : 
     434             :     if (CFP->getValueType(0) == MVT::f64) {
     435             :       // If this target supports 64-bit registers, do a single 64-bit store.
     436           8 :       if (TLI.isTypeLegal(MVT::i64)) {
     437           0 :         SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
     438           0 :                                       zextOrTrunc(64), SDLoc(CFP), MVT::i64);
     439           0 :         return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
     440           0 :                             Alignment, MMOFlags, AAInfo);
     441             :       }
     442             : 
     443           8 :       if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
     444             :         // Otherwise, if the target supports 32-bit registers, use 2 32-bit
     445             :         // stores.  If the target supports neither 32- nor 64-bits, this
     446             :         // xform is certainly not worth it.
     447           0 :         const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
     448           0 :         SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
     449           0 :         SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
     450           0 :         if (DAG.getDataLayout().isBigEndian())
     451             :           std::swap(Lo, Hi);
     452             : 
     453           0 :         Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
     454           0 :                           MMOFlags, AAInfo);
     455           0 :         Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     456           0 :                           DAG.getConstant(4, dl, Ptr.getValueType()));
     457           0 :         Hi = DAG.getStore(Chain, dl, Hi, Ptr,
     458           0 :                           ST->getPointerInfo().getWithOffset(4),
     459           0 :                           MinAlign(Alignment, 4U), MMOFlags, AAInfo);
     460             : 
     461           0 :         return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
     462             :       }
     463             :     }
     464             :   }
     465     1389386 :   return SDValue(nullptr, 0);
     466             : }
     467             : 
     468     1418494 : void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
     469             :   StoreSDNode *ST = cast<StoreSDNode>(Node);
     470     1418494 :   SDValue Chain = ST->getChain();
     471     1418494 :   SDValue Ptr = ST->getBasePtr();
     472             :   SDLoc dl(Node);
     473             : 
     474     1418494 :   unsigned Alignment = ST->getAlignment();
     475     1418494 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
     476             :   AAMDNodes AAInfo = ST->getAAInfo();
     477             : 
     478     1418494 :   if (!ST->isTruncatingStore()) {
     479             :     LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
     480     1389387 :     if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
     481           1 :       ReplaceNode(ST, OptStore);
     482           1 :       return;
     483             :     }
     484             : 
     485     1389386 :     SDValue Value = ST->getValue();
     486             :     MVT VT = Value.getSimpleValueType();
     487     2778772 :     switch (TLI.getOperationAction(ISD::STORE, VT)) {
     488           0 :     default: llvm_unreachable("This action is not supported yet!");
     489     1259312 :     case TargetLowering::Legal: {
     490             :       // If this is an unaligned store and the target doesn't support it,
     491             :       // expand it.
     492     1259312 :       EVT MemVT = ST->getMemoryVT();
     493             :       unsigned AS = ST->getAddressSpace();
     494     1259312 :       unsigned Align = ST->getAlignment();
     495     1259312 :       const DataLayout &DL = DAG.getDataLayout();
     496     1259312 :       if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     497             :         LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
     498         234 :         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
     499         234 :         ReplaceNode(SDValue(ST, 0), Result);
     500             :       } else
     501             :         LLVM_DEBUG(dbgs() << "Legal store\n");
     502             :       break;
     503             :     }
     504      119583 :     case TargetLowering::Custom: {
     505             :       LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
     506      239166 :       SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
     507      119583 :       if (Res && Res != SDValue(Node, 0))
     508       14971 :         ReplaceNode(SDValue(Node, 0), Res);
     509             :       return;
     510             :     }
     511       10491 :     case TargetLowering::Promote: {
     512       10491 :       MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
     513             :       assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
     514             :              "Can only promote stores to same size type");
     515       20982 :       Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
     516             :       SDValue Result =
     517       20982 :           DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     518       20982 :                        Alignment, MMOFlags, AAInfo);
     519       10491 :       ReplaceNode(SDValue(Node, 0), Result);
     520             :       break;
     521             :     }
     522             :     }
     523             :     return;
     524             :   }
     525             : 
     526             :   LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
     527       29107 :   SDValue Value = ST->getValue();
     528       29107 :   EVT StVT = ST->getMemoryVT();
     529       29107 :   unsigned StWidth = StVT.getSizeInBits();
     530       29107 :   auto &DL = DAG.getDataLayout();
     531             : 
     532       29107 :   if (StWidth != StVT.getStoreSizeInBits()) {
     533             :     // Promote to a byte-sized store with upper bits zero if not
     534             :     // storing an integral number of bytes.  For example, promote
     535             :     // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
     536        1802 :     EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
     537        1802 :                                 StVT.getStoreSizeInBits());
     538        1802 :     Value = DAG.getZeroExtendInReg(Value, dl, StVT);
     539             :     SDValue Result =
     540        3604 :         DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
     541        3604 :                           Alignment, MMOFlags, AAInfo);
     542        1802 :     ReplaceNode(SDValue(Node, 0), Result);
     543       27305 :   } else if (StWidth & (StWidth - 1)) {
     544             :     // If not storing a power-of-2 number of bits, expand as two stores.
     545             :     assert(!StVT.isVector() && "Unsupported truncstore!");
     546         710 :     unsigned RoundWidth = 1 << Log2_32(StWidth);
     547             :     assert(RoundWidth < StWidth);
     548         710 :     unsigned ExtraWidth = StWidth - RoundWidth;
     549             :     assert(ExtraWidth < RoundWidth);
     550             :     assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
     551             :            "Store size not an integral number of bytes!");
     552         710 :     EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
     553         710 :     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
     554         710 :     SDValue Lo, Hi;
     555             :     unsigned IncrementSize;
     556             : 
     557         710 :     if (DL.isLittleEndian()) {
     558             :       // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
     559             :       // Store the bottom RoundWidth bits.
     560        2094 :       Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     561         698 :                              RoundVT, Alignment, MMOFlags, AAInfo);
     562             : 
     563             :       // Store the remaining ExtraWidth bits.
     564         698 :       IncrementSize = RoundWidth / 8;
     565        1396 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     566             :                         DAG.getConstant(IncrementSize, dl,
     567        1396 :                                         Ptr.getValueType()));
     568        1396 :       Hi = DAG.getNode(
     569             :           ISD::SRL, dl, Value.getValueType(), Value,
     570             :           DAG.getConstant(RoundWidth, dl,
     571        2094 :                           TLI.getShiftAmountTy(Value.getValueType(), DL)));
     572        1396 :       Hi = DAG.getTruncStore(
     573             :           Chain, dl, Hi, Ptr,
     574         698 :           ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
     575        1396 :           MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
     576             :     } else {
     577             :       // Big endian - avoid unaligned stores.
     578             :       // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
     579             :       // Store the top RoundWidth bits.
     580          24 :       Hi = DAG.getNode(
     581             :           ISD::SRL, dl, Value.getValueType(), Value,
     582             :           DAG.getConstant(ExtraWidth, dl,
     583          36 :                           TLI.getShiftAmountTy(Value.getValueType(), DL)));
     584          36 :       Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
     585          12 :                              RoundVT, Alignment, MMOFlags, AAInfo);
     586             : 
     587             :       // Store the remaining ExtraWidth bits.
     588          12 :       IncrementSize = RoundWidth / 8;
     589          24 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     590             :                         DAG.getConstant(IncrementSize, dl,
     591          24 :                                         Ptr.getValueType()));
     592          24 :       Lo = DAG.getTruncStore(
     593             :           Chain, dl, Value, Ptr,
     594          12 :           ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
     595          24 :           MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
     596             :     }
     597             : 
     598             :     // The order of the stores doesn't matter.
     599        1420 :     SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
     600         710 :     ReplaceNode(SDValue(Node, 0), Result);
     601             :   } else {
     602      106380 :     switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
     603           0 :     default: llvm_unreachable("This action is not supported yet!");
     604       23320 :     case TargetLowering::Legal: {
     605       23320 :       EVT MemVT = ST->getMemoryVT();
     606             :       unsigned AS = ST->getAddressSpace();
     607       23320 :       unsigned Align = ST->getAlignment();
     608             :       // If this is an unaligned store and the target doesn't support it,
     609             :       // expand it.
     610       23320 :       if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     611         454 :         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
     612         454 :         ReplaceNode(SDValue(ST, 0), Result);
     613             :       }
     614             :       break;
     615             :     }
     616        1700 :     case TargetLowering::Custom: {
     617        3400 :       SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
     618        1700 :       if (Res && Res != SDValue(Node, 0))
     619        1538 :         ReplaceNode(SDValue(Node, 0), Res);
     620             :       return;
     621             :     }
     622        1575 :     case TargetLowering::Expand:
     623             :       assert(!StVT.isVector() &&
     624             :              "Vector Stores are handled in LegalizeVectorOps");
     625             : 
     626        1575 :       SDValue Result;
     627             : 
     628             :       // TRUNCSTORE:i16 i32 -> STORE i16
     629             :       if (TLI.isTypeLegal(StVT)) {
     630        3126 :         Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
     631        4689 :         Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     632        1563 :                               Alignment, MMOFlags, AAInfo);
     633             :       } else {
     634             :         // The in-memory type isn't legal. Truncate to the type it would promote
     635             :         // to, and then do a truncstore.
     636          24 :         Value = DAG.getNode(ISD::TRUNCATE, dl,
     637          12 :                             TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
     638          24 :                             Value);
     639          36 :         Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     640          12 :                                    StVT, Alignment, MMOFlags, AAInfo);
     641             :       }
     642             : 
     643        1575 :       ReplaceNode(SDValue(Node, 0), Result);
     644        1575 :       break;
     645             :     }
     646             :   }
     647             : }
     648             : 
     649     1219002 : void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
     650             :   LoadSDNode *LD = cast<LoadSDNode>(Node);
     651     1219002 :   SDValue Chain = LD->getChain();  // The chain.
     652     1219002 :   SDValue Ptr = LD->getBasePtr();  // The base pointer.
     653     1219002 :   SDValue Value;                   // The value returned by the load op.
     654             :   SDLoc dl(Node);
     655             : 
     656             :   ISD::LoadExtType ExtType = LD->getExtensionType();
     657     1219002 :   if (ExtType == ISD::NON_EXTLOAD) {
     658             :     LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
     659             :     MVT VT = Node->getSimpleValueType(0);
     660             :     SDValue RVal = SDValue(Node, 0);
     661             :     SDValue RChain = SDValue(Node, 1);
     662             : 
     663     2330938 :     switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
     664           0 :     default: llvm_unreachable("This action is not supported yet!");
     665      963500 :     case TargetLowering::Legal: {
     666      963500 :       EVT MemVT = LD->getMemoryVT();
     667             :       unsigned AS = LD->getAddressSpace();
     668      963500 :       unsigned Align = LD->getAlignment();
     669      963500 :       const DataLayout &DL = DAG.getDataLayout();
     670             :       // If this is an unaligned load and the target doesn't support it,
     671             :       // expand it.
     672      963500 :       if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     673         382 :         std::tie(RVal, RChain) =  TLI.expandUnalignedLoad(LD, DAG);
     674             :       }
     675             :       break;
     676             :     }
     677      140504 :     case TargetLowering::Custom:
     678      140504 :       if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
     679       21588 :         RVal = Res;
     680       21588 :         RChain = Res.getValue(1);
     681      140504 :       }
     682      140504 :       break;
     683             : 
     684       61465 :     case TargetLowering::Promote: {
     685       61465 :       MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
     686             :       assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
     687             :              "Can only promote loads to same size type");
     688             : 
     689      122930 :       SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
     690      122930 :       RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
     691       61465 :       RChain = Res.getValue(1);
     692             :       break;
     693             :     }
     694             :     }
     695     1165469 :     if (RChain.getNode() != Node) {
     696             :       assert(RVal.getNode() != Node && "Load must be completely replaced");
     697      152584 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
     698      152584 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
     699       76292 :       if (UpdatedNodes) {
     700        2668 :         UpdatedNodes->insert(RVal.getNode());
     701        2668 :         UpdatedNodes->insert(RChain.getNode());
     702             :       }
     703       76292 :       ReplacedNode(Node);
     704             :     }
     705             :     return;
     706             :   }
     707             : 
     708             :   LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
     709       53533 :   EVT SrcVT = LD->getMemoryVT();
     710       53533 :   unsigned SrcWidth = SrcVT.getSizeInBits();
     711       53533 :   unsigned Alignment = LD->getAlignment();
     712       53533 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
     713             :   AAMDNodes AAInfo = LD->getAAInfo();
     714             : 
     715       53533 :   if (SrcWidth != SrcVT.getStoreSizeInBits() &&
     716             :       // Some targets pretend to have an i1 loading operation, and actually
     717             :       // load an i8.  This trick is correct for ZEXTLOAD because the top 7
     718             :       // bits are guaranteed to be zero; it helps the optimizers understand
     719             :       // that these bits are zero.  It is also useful for EXTLOAD, since it
     720             :       // tells the optimizers that those bits are undefined.  It would be
     721             :       // nice to have an effective generic way of getting these benefits...
     722             :       // Until such a way is found, don't insist on promoting i1 here.
     723        1285 :       (SrcVT != MVT::i1 ||
     724         505 :        TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
     725             :          TargetLowering::Promote)) {
     726             :     // Promote to a byte-sized load if not loading an integral number of
     727             :     // bytes.  For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
     728             :     unsigned NewWidth = SrcVT.getStoreSizeInBits();
     729         523 :     EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
     730             :     SDValue Ch;
     731             : 
     732             :     // The extra bits are guaranteed to be zero, since we stored them that
     733             :     // way.  A zext load from NVT thus automatically gives zext from SrcVT.
     734             : 
     735             :     ISD::LoadExtType NewExtType =
     736         523 :       ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
     737             : 
     738             :     SDValue Result =
     739         523 :         DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
     740        1569 :                        LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
     741             : 
     742             :     Ch = Result.getValue(1); // The chain.
     743             : 
     744         523 :     if (ExtType == ISD::SEXTLOAD)
     745             :       // Having the top bits zero doesn't help when sign extending.
     746         184 :       Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
     747             :                            Result.getValueType(),
     748         184 :                            Result, DAG.getValueType(SrcVT));
     749         431 :     else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
     750             :       // All the top bits are guaranteed to be zero - inform the optimizers.
     751         476 :       Result = DAG.getNode(ISD::AssertZext, dl,
     752             :                            Result.getValueType(), Result,
     753         476 :                            DAG.getValueType(SrcVT));
     754             : 
     755         523 :     Value = Result;
     756         523 :     Chain = Ch;
     757       53010 :   } else if (SrcWidth & (SrcWidth - 1)) {
     758             :     // If not loading a power-of-2 number of bits, expand as two loads.
     759             :     assert(!SrcVT.isVector() && "Unsupported extload!");
     760         254 :     unsigned RoundWidth = 1 << Log2_32(SrcWidth);
     761             :     assert(RoundWidth < SrcWidth);
     762         254 :     unsigned ExtraWidth = SrcWidth - RoundWidth;
     763             :     assert(ExtraWidth < RoundWidth);
     764             :     assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
     765             :            "Load size not an integral number of bytes!");
     766         254 :     EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
     767         254 :     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
     768         254 :     SDValue Lo, Hi, Ch;
     769             :     unsigned IncrementSize;
     770         254 :     auto &DL = DAG.getDataLayout();
     771             : 
     772         254 :     if (DL.isLittleEndian()) {
     773             :       // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
     774             :       // Load the bottom RoundWidth bits.
     775         458 :       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
     776         229 :                           LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
     777         458 :                           AAInfo);
     778             : 
     779             :       // Load the remaining ExtraWidth bits.
     780         229 :       IncrementSize = RoundWidth / 8;
     781         458 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     782             :                          DAG.getConstant(IncrementSize, dl,
     783         458 :                                          Ptr.getValueType()));
     784         458 :       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
     785         229 :                           LD->getPointerInfo().getWithOffset(IncrementSize),
     786             :                           ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
     787         458 :                           AAInfo);
     788             : 
     789             :       // Build a factor node to remember that this load is independent of
     790             :       // the other one.
     791         458 :       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
     792         229 :                        Hi.getValue(1));
     793             : 
     794             :       // Move the top bits to the right place.
     795         458 :       Hi = DAG.getNode(
     796             :           ISD::SHL, dl, Hi.getValueType(), Hi,
     797             :           DAG.getConstant(RoundWidth, dl,
     798         687 :                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
     799             : 
     800             :       // Join the hi and lo parts.
     801         687 :       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
     802             :     } else {
     803             :       // Big endian - avoid unaligned loads.
     804             :       // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
     805             :       // Load the top RoundWidth bits.
     806          50 :       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
     807          25 :                           LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
     808          50 :                           AAInfo);
     809             : 
     810             :       // Load the remaining ExtraWidth bits.
     811          25 :       IncrementSize = RoundWidth / 8;
     812          50 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     813             :                          DAG.getConstant(IncrementSize, dl,
     814          50 :                                          Ptr.getValueType()));
     815          50 :       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
     816          25 :                           LD->getPointerInfo().getWithOffset(IncrementSize),
     817             :                           ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
     818          50 :                           AAInfo);
     819             : 
     820             :       // Build a factor node to remember that this load is independent of
     821             :       // the other one.
     822          50 :       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
     823          25 :                        Hi.getValue(1));
     824             : 
     825             :       // Move the top bits to the right place.
     826          50 :       Hi = DAG.getNode(
     827             :           ISD::SHL, dl, Hi.getValueType(), Hi,
     828             :           DAG.getConstant(ExtraWidth, dl,
     829          75 :                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
     830             : 
     831             :       // Join the hi and lo parts.
     832          75 :       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
     833             :     }
     834             : 
     835         254 :     Chain = Ch;
     836             :   } else {
     837             :     bool isCustom = false;
     838      105512 :     switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
     839             :                                  SrcVT.getSimpleVT())) {
     840           0 :     default: llvm_unreachable("This action is not supported yet!");
     841        6047 :     case TargetLowering::Custom:
     842             :       isCustom = true;
     843             :       LLVM_FALLTHROUGH;
     844       52374 :     case TargetLowering::Legal:
     845       52374 :       Value = SDValue(Node, 0);
     846       52374 :       Chain = SDValue(Node, 1);
     847             : 
     848       52374 :       if (isCustom) {
     849       12094 :         if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
     850        4391 :           Value = Res;
     851        4391 :           Chain = Res.getValue(1);
     852             :         }
     853             :       } else {
     854             :         // If this is an unaligned load and the target doesn't support it,
     855             :         // expand it.
     856       46327 :         EVT MemVT = LD->getMemoryVT();
     857             :         unsigned AS = LD->getAddressSpace();
     858       46327 :         unsigned Align = LD->getAlignment();
     859       46327 :         const DataLayout &DL = DAG.getDataLayout();
     860       46327 :         if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     861         940 :           std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
     862             :         }
     863             :       }
     864             :       break;
     865             : 
     866         382 :     case TargetLowering::Expand: {
     867         382 :       EVT DestVT = Node->getValueType(0);
     868         382 :       if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
     869             :         // If the source type is not legal, see if there is a legal extload to
     870             :         // an intermediate type that we can then extend further.
     871             :         EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
     872          54 :         if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
     873             :             TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
     874             :           // If we are loading a legal type, this is a non-extload followed by a
     875             :           // full extend.
     876             :           ISD::LoadExtType MidExtType =
     877         362 :               (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
     878             : 
     879         362 :           SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
     880         362 :                                         SrcVT, LD->getMemOperand());
     881             :           unsigned ExtendOp =
     882         362 :               ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
     883        1086 :           Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
     884         362 :           Chain = Load.getValue(1);
     885             :           break;
     886             :         }
     887             : 
     888             :         // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
     889             :         // normal undefined upper bits behavior to allow using an in-reg extend
     890             :         // with the illegal FP type, so load as an integer and do the
     891             :         // from-integer conversion.
     892           0 :         if (SrcVT.getScalarType() == MVT::f16) {
     893           0 :           EVT ISrcVT = SrcVT.changeTypeToInteger();
     894           0 :           EVT IDestVT = DestVT.changeTypeToInteger();
     895           0 :           EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
     896             : 
     897           0 :           SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
     898             :                                           Chain, Ptr, ISrcVT,
     899           0 :                                           LD->getMemOperand());
     900           0 :           Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
     901           0 :           Chain = Result.getValue(1);
     902             :           break;
     903             :         }
     904             :       }
     905             : 
     906             :       assert(!SrcVT.isVector() &&
     907             :              "Vector Loads are handled in LegalizeVectorOps");
     908             : 
     909             :       // FIXME: This does not work for vectors on most targets.  Sign-
     910             :       // and zero-extend operations are currently folded into extending
     911             :       // loads, whether they are legal or not, and then we end up here
     912             :       // without any support for legalizing them.
     913             :       assert(ExtType != ISD::EXTLOAD &&
     914             :              "EXTLOAD should always be supported!");
     915             :       // Turn the unsupported load into an EXTLOAD followed by an
     916             :       // explicit zero/sign extend inreg.
     917          20 :       SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
     918             :                                       Node->getValueType(0),
     919             :                                       Chain, Ptr, SrcVT,
     920          40 :                                       LD->getMemOperand());
     921             :       SDValue ValRes;
     922          20 :       if (ExtType == ISD::SEXTLOAD)
     923          38 :         ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
     924             :                              Result.getValueType(),
     925          38 :                              Result, DAG.getValueType(SrcVT));
     926             :       else
     927           1 :         ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
     928          20 :       Value = ValRes;
     929          20 :       Chain = Result.getValue(1);
     930          20 :       break;
     931             :     }
     932             :     }
     933             :   }
     934             : 
     935             :   // Since loads produce two values, make sure to remember that we legalized
     936             :   // both of them.
     937       53533 :   if (Chain.getNode() != Node) {
     938             :     assert(Value.getNode() != Node && "Load must be completely replaced");
     939       11996 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
     940       11996 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
     941        5998 :     if (UpdatedNodes) {
     942           1 :       UpdatedNodes->insert(Value.getNode());
     943           1 :       UpdatedNodes->insert(Chain.getNode());
     944             :     }
     945        5998 :     ReplacedNode(Node);
     946             :   }
     947             : }
     948             : 
     949             : /// Return a legal replacement for the given operation, with all legal operands.
     950    23483159 : void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     951             :   LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
     952             : 
     953             :   // Allow illegal target nodes and illegal registers.
     954    46966318 :   if (Node->getOpcode() == ISD::TargetConstant ||
     955             :       Node->getOpcode() == ISD::Register)
     956             :     return;
     957             : 
     958             : #ifndef NDEBUG
     959             :   for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
     960             :     assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
     961             :               TargetLowering::TypeLegal ||
     962             :             TLI.isTypeLegal(Node->getValueType(i))) &&
     963             :            "Unexpected illegal type!");
     964             : 
     965             :   for (const SDValue &Op : Node->op_values())
     966             :     assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
     967             :               TargetLowering::TypeLegal ||
     968             :             TLI.isTypeLegal(Op.getValueType()) ||
     969             :             Op.getOpcode() == ISD::TargetConstant ||
     970             :             Op.getOpcode() == ISD::Register) &&
     971             :             "Unexpected illegal type!");
     972             : #endif
     973             : 
     974             :   // Figure out the correct action; the way to query this varies by opcode
     975             :   TargetLowering::LegalizeAction Action = TargetLowering::Legal;
     976             :   bool SimpleFinishLegalizing = true;
     977    20588025 :   switch (Node->getOpcode()) {
     978       70296 :   case ISD::INTRINSIC_W_CHAIN:
     979             :   case ISD::INTRINSIC_WO_CHAIN:
     980             :   case ISD::INTRINSIC_VOID:
     981             :   case ISD::STACKSAVE:
     982       70296 :     Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
     983             :     break;
     984           4 :   case ISD::GET_DYNAMIC_AREA_OFFSET:
     985           4 :     Action = TLI.getOperationAction(Node->getOpcode(),
     986             :                                     Node->getValueType(0));
     987             :     break;
     988         197 :   case ISD::VAARG:
     989         197 :     Action = TLI.getOperationAction(Node->getOpcode(),
     990             :                                     Node->getValueType(0));
     991         197 :     if (Action != TargetLowering::Promote)
     992             :       Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
     993             :     break;
     994      257796 :   case ISD::FP_TO_FP16:
     995             :   case ISD::SINT_TO_FP:
     996             :   case ISD::UINT_TO_FP:
     997             :   case ISD::EXTRACT_VECTOR_ELT:
     998      515592 :     Action = TLI.getOperationAction(Node->getOpcode(),
     999      257796 :                                     Node->getOperand(0).getValueType());
    1000             :     break;
    1001       27492 :   case ISD::FP_ROUND_INREG:
    1002             :   case ISD::SIGN_EXTEND_INREG: {
    1003       27492 :     EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
    1004       27492 :     Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
    1005             :     break;
    1006             :   }
    1007        1493 :   case ISD::ATOMIC_STORE:
    1008        2986 :     Action = TLI.getOperationAction(Node->getOpcode(),
    1009        1493 :                                     Node->getOperand(2).getValueType());
    1010             :     break;
    1011       76650 :   case ISD::SELECT_CC:
    1012             :   case ISD::SETCC:
    1013             :   case ISD::BR_CC: {
    1014       76650 :     unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
    1015             :                          Node->getOpcode() == ISD::SETCC ? 2 : 1;
    1016       76650 :     unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
    1017       76650 :     MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
    1018             :     ISD::CondCode CCCode =
    1019       76650 :         cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
    1020       76650 :     Action = TLI.getCondCodeAction(CCCode, OpVT);
    1021       76650 :     if (Action == TargetLowering::Legal) {
    1022       76012 :       if (Node->getOpcode() == ISD::SELECT_CC)
    1023       22075 :         Action = TLI.getOperationAction(Node->getOpcode(),
    1024             :                                         Node->getValueType(0));
    1025             :       else
    1026             :         Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
    1027             :     }
    1028             :     break;
    1029             :   }
    1030             :   case ISD::LOAD:
    1031             :   case ISD::STORE:
    1032             :     // FIXME: Model these properly.  LOAD and STORE are complicated, and
    1033             :     // STORE expects the unlegalized operand in some cases.
    1034             :     SimpleFinishLegalizing = false;
    1035             :     break;
    1036             :   case ISD::CALLSEQ_START:
    1037             :   case ISD::CALLSEQ_END:
    1038             :     // FIXME: This shouldn't be necessary.  These nodes have special properties
    1039             :     // dealing with the recursive nature of legalization.  Removing this
    1040             :     // special case should be done as part of making LegalizeDAG non-recursive.
    1041             :     SimpleFinishLegalizing = false;
    1042             :     break;
    1043      155924 :   case ISD::EXTRACT_ELEMENT:
    1044             :   case ISD::FLT_ROUNDS_:
    1045             :   case ISD::MERGE_VALUES:
    1046             :   case ISD::EH_RETURN:
    1047             :   case ISD::FRAME_TO_ARGS_OFFSET:
    1048             :   case ISD::EH_DWARF_CFA:
    1049             :   case ISD::EH_SJLJ_SETJMP:
    1050             :   case ISD::EH_SJLJ_LONGJMP:
    1051             :   case ISD::EH_SJLJ_SETUP_DISPATCH:
    1052             :     // These operations lie about being legal: when they claim to be legal,
    1053             :     // they should actually be expanded.
    1054      155924 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1055      155924 :     if (Action == TargetLowering::Legal)
    1056             :       Action = TargetLowering::Expand;
    1057             :     break;
    1058         214 :   case ISD::INIT_TRAMPOLINE:
    1059             :   case ISD::ADJUST_TRAMPOLINE:
    1060             :   case ISD::FRAMEADDR:
    1061             :   case ISD::RETURNADDR:
    1062             :   case ISD::ADDROFRETURNADDR:
    1063             :     // These operations lie about being legal: when they claim to be legal,
    1064             :     // they should actually be custom-lowered.
    1065         214 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1066         214 :     if (Action == TargetLowering::Legal)
    1067             :       Action = TargetLowering::Custom;
    1068             :     break;
    1069          25 :   case ISD::READCYCLECOUNTER:
    1070             :     // READCYCLECOUNTER returns an i64, even if type legalization might have
    1071             :     // expanded that to several smaller types.
    1072          25 :     Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
    1073             :     break;
    1074             :   case ISD::READ_REGISTER:
    1075             :   case ISD::WRITE_REGISTER:
    1076             :     // Named register is legal in the DAG, but blocked by register name
    1077             :     // selection if not implemented by target (to chose the correct register)
    1078             :     // They'll be converted to Copy(To/From)Reg.
    1079             :     Action = TargetLowering::Legal;
    1080             :     break;
    1081          28 :   case ISD::DEBUGTRAP:
    1082          28 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1083          28 :     if (Action == TargetLowering::Expand) {
    1084             :       // replace ISD::DEBUGTRAP with ISD::TRAP
    1085             :       SDValue NewVal;
    1086          39 :       NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
    1087          26 :                            Node->getOperand(0));
    1088          13 :       ReplaceNode(Node, NewVal.getNode());
    1089          13 :       LegalizeOp(NewVal.getNode());
    1090             :       return;
    1091          15 :     }
    1092             :     break;
    1093         132 :   case ISD::STRICT_FADD:
    1094             :   case ISD::STRICT_FSUB:
    1095             :   case ISD::STRICT_FMUL:
    1096             :   case ISD::STRICT_FDIV:
    1097             :   case ISD::STRICT_FSQRT:
    1098             :   case ISD::STRICT_FMA:
    1099             :   case ISD::STRICT_FPOW:
    1100             :   case ISD::STRICT_FPOWI:
    1101             :   case ISD::STRICT_FSIN:
    1102             :   case ISD::STRICT_FCOS:
    1103             :   case ISD::STRICT_FEXP:
    1104             :   case ISD::STRICT_FEXP2:
    1105             :   case ISD::STRICT_FLOG:
    1106             :   case ISD::STRICT_FLOG10:
    1107             :   case ISD::STRICT_FLOG2:
    1108             :   case ISD::STRICT_FRINT:
    1109             :   case ISD::STRICT_FNEARBYINT:
    1110             :     // These pseudo-ops get legalized as if they were their non-strict
    1111             :     // equivalent.  For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
    1112             :     // is also legal, but if ISD::FSQRT requires expansion then so does
    1113             :     // ISD::STRICT_FSQRT.
    1114         264 :     Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
    1115             :                                             Node->getValueType(0));
    1116             :     break;
    1117    16617220 :   default:
    1118    16617220 :     if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
    1119             :       Action = TargetLowering::Legal;
    1120             :     } else {
    1121    13639991 :       Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1122             :     }
    1123             :     break;
    1124             :   }
    1125             : 
    1126             :   if (SimpleFinishLegalizing) {
    1127             :     SDNode *NewNode = Node;
    1128    34416424 :     switch (Node->getOpcode()) {
    1129             :     default: break;
    1130      284511 :     case ISD::SHL:
    1131             :     case ISD::SRL:
    1132             :     case ISD::SRA:
    1133             :     case ISD::ROTL:
    1134             :     case ISD::ROTR: {
    1135             :       // Legalizing shifts/rotates requires adjusting the shift amount
    1136             :       // to the appropriate width.
    1137      284511 :       SDValue Op0 = Node->getOperand(0);
    1138      284511 :       SDValue Op1 = Node->getOperand(1);
    1139      569022 :       if (!Op1.getValueType().isVector()) {
    1140      561716 :         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
    1141             :         // The getShiftAmountOperand() may create a new operand node or
    1142             :         // return the existing one. If new operand is created we need
    1143             :         // to update the parent node.
    1144             :         // Do not try to legalize SAO here! It will be automatically legalized
    1145             :         // in the next round.
    1146             :         if (SAO != Op1)
    1147       42525 :           NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
    1148      284511 :       }
    1149             :     }
    1150      284511 :     break;
    1151         552 :     case ISD::SRL_PARTS:
    1152             :     case ISD::SRA_PARTS:
    1153             :     case ISD::SHL_PARTS: {
    1154             :       // Legalizing shifts/rotates requires adjusting the shift amount
    1155             :       // to the appropriate width.
    1156         552 :       SDValue Op0 = Node->getOperand(0);
    1157         552 :       SDValue Op1 = Node->getOperand(1);
    1158         552 :       SDValue Op2 = Node->getOperand(2);
    1159        1104 :       if (!Op2.getValueType().isVector()) {
    1160        1104 :         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
    1161             :         // The getShiftAmountOperand() may create a new operand node or
    1162             :         // return the existing one. If new operand is created we need
    1163             :         // to update the parent node.
    1164             :         if (SAO != Op2)
    1165           0 :           NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
    1166             :       }
    1167             :       break;
    1168             :     }
    1169             :     }
    1170             : 
    1171    17208212 :     if (NewNode != Node) {
    1172         128 :       ReplaceNode(Node, NewNode);
    1173             :       Node = NewNode;
    1174             :     }
    1175    17208212 :     switch (Action) {
    1176             :     case TargetLowering::Legal:
    1177             :       LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
    1178             :       return;
    1179     1110778 :     case TargetLowering::Custom:
    1180             :       LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
    1181             :       // FIXME: The handling for custom lowering with multiple results is
    1182             :       // a complete mess.
    1183     2221556 :       if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
    1184     1056296 :         if (!(Res.getNode() != Node || Res.getResNo() != 0))
    1185             :           return;
    1186             : 
    1187      821981 :         if (Node->getNumValues() == 1) {
    1188             :           LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
    1189             :           // We can just directly replace this node with the lowered value.
    1190      678538 :           ReplaceNode(SDValue(Node, 0), Res);
    1191      678538 :           return;
    1192             :         }
    1193             : 
    1194             :         SmallVector<SDValue, 8> ResultVals;
    1195      860786 :         for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
    1196      286950 :           ResultVals.push_back(Res.getValue(i));
    1197             :         LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
    1198      143443 :         ReplaceNode(Node, ResultVals.data());
    1199             :         return;
    1200     1056296 :       }
    1201             :       LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
    1202             :       LLVM_FALLTHROUGH;
    1203             :     case TargetLowering::Expand:
    1204      239508 :       if (ExpandNode(Node))
    1205             :         return;
    1206             :       LLVM_FALLTHROUGH;
    1207             :     case TargetLowering::LibCall:
    1208       42049 :       ConvertNodeToLibcall(Node);
    1209       42049 :       return;
    1210        7227 :     case TargetLowering::Promote:
    1211        7227 :       PromoteNode(Node);
    1212        7227 :       return;
    1213             :     }
    1214             :   }
    1215             : 
    1216     6759600 :   switch (Node->getOpcode()) {
    1217           0 :   default:
    1218             : #ifndef NDEBUG
    1219             :     dbgs() << "NODE: ";
    1220             :     Node->dump( &DAG);
    1221             :     dbgs() << "\n";
    1222             : #endif
    1223           0 :     llvm_unreachable("Do not know how to legalize this operator!");
    1224             : 
    1225             :   case ISD::CALLSEQ_START:
    1226             :   case ISD::CALLSEQ_END:
    1227             :     break;
    1228     1219002 :   case ISD::LOAD:
    1229     1219002 :     return LegalizeLoadOps(Node);
    1230     1418494 :   case ISD::STORE:
    1231     1418494 :     return LegalizeStoreOps(Node);
    1232             :   }
    1233             : }
    1234             : 
    1235        1790 : SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
    1236        1790 :   SDValue Vec = Op.getOperand(0);
    1237        1790 :   SDValue Idx = Op.getOperand(1);
    1238             :   SDLoc dl(Op);
    1239             : 
    1240             :   // Before we generate a new store to a temporary stack slot, see if there is
    1241             :   // already one that we can use. There often is because when we scalarize
    1242             :   // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
    1243             :   // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
    1244             :   // the vector. If all are expanded here, we don't want one store per vector
    1245             :   // element.
    1246             : 
    1247             :   // Caches for hasPredecessorHelper
    1248             :   SmallPtrSet<const SDNode *, 32> Visited;
    1249             :   SmallVector<const SDNode *, 16> Worklist;
    1250        1790 :   Worklist.push_back(Idx.getNode());
    1251        1790 :   SDValue StackPtr, Ch;
    1252        1790 :   for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
    1253        3608 :        UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
    1254             :     SDNode *User = *UI;
    1255             :     if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
    1256        2676 :       if (ST->isIndexed() || ST->isTruncatingStore() ||
    1257             :           ST->getValue() != Vec)
    1258             :         continue;
    1259             : 
    1260             :       // Make sure that nothing else could have stored into the destination of
    1261             :       // this store.
    1262        2676 :       if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
    1263             :         continue;
    1264             : 
    1265             :       // If the index is dependent on the store we will introduce a cycle when
    1266             :       // creating the load (the load uses the index, and by replacing the chain
    1267             :       // we will make the index dependent on the load). Also, the store might be
    1268             :       // dependent on the extractelement and introduce a cycle when creating
    1269             :       // the load.
    1270        2677 :       if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
    1271        1337 :           ST->hasPredecessor(Op.getNode()))
    1272             :         continue;
    1273             : 
    1274        1336 :       StackPtr = ST->getBasePtr();
    1275             :       Ch = SDValue(ST, 0);
    1276             :       break;
    1277             :     }
    1278             :   }
    1279             : 
    1280        1790 :   EVT VecVT = Vec.getValueType();
    1281             : 
    1282        1790 :   if (!Ch.getNode()) {
    1283             :     // Store the value to a temporary stack slot, then LOAD the returned part.
    1284         454 :     StackPtr = DAG.CreateStackTemporary(VecVT);
    1285         908 :     Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
    1286         454 :                       MachinePointerInfo());
    1287             :   }
    1288             : 
    1289        1790 :   StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1290             : 
    1291             :   SDValue NewLoad;
    1292             : 
    1293        3580 :   if (Op.getValueType().isVector())
    1294           0 :     NewLoad =
    1295           0 :         DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
    1296             :   else
    1297        3580 :     NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
    1298             :                              MachinePointerInfo(),
    1299        3580 :                              VecVT.getVectorElementType());
    1300             : 
    1301             :   // Replace the chain going out of the store, by the one out of the load.
    1302        3580 :   DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
    1303             : 
    1304             :   // We introduced a cycle though, so update the loads operands, making sure
    1305             :   // to use the original store's chain as an incoming chain.
    1306             :   SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
    1307        1790 :                                           NewLoad->op_end());
    1308        1790 :   NewLoadOperands[0] = Ch;
    1309             :   NewLoad =
    1310        3580 :       SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
    1311        3580 :   return NewLoad;
    1312             : }
    1313             : 
    1314           0 : SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
    1315             :   assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
    1316             : 
    1317           0 :   SDValue Vec  = Op.getOperand(0);
    1318           0 :   SDValue Part = Op.getOperand(1);
    1319           0 :   SDValue Idx  = Op.getOperand(2);
    1320             :   SDLoc dl(Op);
    1321             : 
    1322             :   // Store the value to a temporary stack slot, then LOAD the returned part.
    1323           0 :   EVT VecVT = Vec.getValueType();
    1324           0 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1325           0 :   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1326             :   MachinePointerInfo PtrInfo =
    1327           0 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
    1328             : 
    1329             :   // First store the whole vector.
    1330           0 :   SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
    1331             : 
    1332             :   // Then store the inserted part.
    1333           0 :   SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1334             : 
    1335             :   // Store the subvector.
    1336           0 :   Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
    1337             : 
    1338             :   // Finally, load the updated vector.
    1339           0 :   return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
    1340             : }
    1341             : 
    1342         151 : SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
    1343             :   // We can't handle this case efficiently.  Allocate a sufficiently
    1344             :   // aligned object on the stack, store each element into it, then load
    1345             :   // the result as a vector.
    1346             :   // Create the stack frame object.
    1347         302 :   EVT VT = Node->getValueType(0);
    1348         151 :   EVT EltVT = VT.getVectorElementType();
    1349             :   SDLoc dl(Node);
    1350         151 :   SDValue FIPtr = DAG.CreateStackTemporary(VT);
    1351         151 :   int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
    1352             :   MachinePointerInfo PtrInfo =
    1353         151 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
    1354             : 
    1355             :   // Emit a store of each element to the stack slot.
    1356             :   SmallVector<SDValue, 8> Stores;
    1357         151 :   unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
    1358             :   // Store (in the right endianness) the elements to memory.
    1359        1342 :   for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
    1360             :     // Ignore undef elements.
    1361        1560 :     if (Node->getOperand(i).isUndef()) continue;
    1362             : 
    1363         383 :     unsigned Offset = TypeByteSize*i;
    1364             : 
    1365         766 :     SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
    1366         766 :     Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
    1367             : 
    1368             :     // If the destination vector element type is narrower than the source
    1369             :     // element type, only store the bits necessary.
    1370        1149 :     if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
    1371         282 :       Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
    1372          94 :                                          Node->getOperand(i), Idx,
    1373         188 :                                          PtrInfo.getWithOffset(Offset), EltVT));
    1374             :     } else
    1375        1156 :       Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
    1376         578 :                                     Idx, PtrInfo.getWithOffset(Offset)));
    1377             :   }
    1378             : 
    1379         151 :   SDValue StoreChain;
    1380         151 :   if (!Stores.empty())    // Not all undef elements?
    1381         302 :     StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
    1382             :   else
    1383           0 :     StoreChain = DAG.getEntryNode();
    1384             : 
    1385             :   // Result is a load from the stack slot.
    1386         453 :   return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
    1387             : }
    1388             : 
    1389             : /// Bitcast a floating-point value to an integer value. Only bitcast the part
    1390             : /// containing the sign bit if the target has no integer value capable of
    1391             : /// holding all bits of the floating-point value.
    1392          79 : void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
    1393             :                                              const SDLoc &DL,
    1394             :                                              SDValue Value) const {
    1395         158 :   EVT FloatVT = Value.getValueType();
    1396          79 :   unsigned NumBits = FloatVT.getSizeInBits();
    1397          79 :   State.FloatVT = FloatVT;
    1398          79 :   EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
    1399             :   // Convert to an integer of the same size.
    1400          79 :   if (TLI.isTypeLegal(IVT)) {
    1401         150 :     State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
    1402          75 :     State.SignMask = APInt::getSignMask(NumBits);
    1403          75 :     State.SignBit = NumBits - 1;
    1404          75 :     return;
    1405             :   }
    1406             : 
    1407           4 :   auto &DataLayout = DAG.getDataLayout();
    1408             :   // Store the float to memory, then load the sign part out as an integer.
    1409           8 :   MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
    1410             :   // First create a temporary that is aligned for both the load and store.
    1411           8 :   SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
    1412           4 :   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1413             :   // Then store the float to it.
    1414           4 :   State.FloatPtr = StackPtr;
    1415           4 :   MachineFunction &MF = DAG.getMachineFunction();
    1416           4 :   State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
    1417           8 :   State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
    1418           4 :                              State.FloatPointerInfo);
    1419             : 
    1420             :   SDValue IntPtr;
    1421           4 :   if (DataLayout.isBigEndian()) {
    1422             :     assert(FloatVT.isByteSized() && "Unsupported floating point type!");
    1423             :     // Load out a legal integer with the same sign bit as the float.
    1424             :     IntPtr = StackPtr;
    1425           1 :     State.IntPointerInfo = State.FloatPointerInfo;
    1426             :   } else {
    1427             :     // Advance the pointer so that the loaded byte will contain the sign bit.
    1428           3 :     unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
    1429           6 :     IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
    1430           6 :                       DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
    1431           3 :     State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
    1432           3 :                                                              ByteOffset);
    1433             :   }
    1434             : 
    1435           4 :   State.IntPtr = IntPtr;
    1436           8 :   State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
    1437           4 :                                   State.IntPointerInfo, MVT::i8);
    1438           8 :   State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
    1439           4 :   State.SignBit = 7;
    1440             : }
    1441             : 
    1442             : /// Replace the integer value produced by getSignAsIntValue() with a new value
    1443             : /// and cast the result back to a floating-point type.
    1444          37 : SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
    1445             :                                               const SDLoc &DL,
    1446             :                                               SDValue NewIntValue) const {
    1447          37 :   if (!State.Chain)
    1448          72 :     return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
    1449             : 
    1450             :   // Override the part containing the sign bit in the value stored on the stack.
    1451           1 :   SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
    1452           1 :                                     State.IntPointerInfo, MVT::i8);
    1453           1 :   return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
    1454           1 :                      State.FloatPointerInfo);
    1455             : }
    1456             : 
    1457          42 : SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
    1458             :   SDLoc DL(Node);
    1459          42 :   SDValue Mag = Node->getOperand(0);
    1460          42 :   SDValue Sign = Node->getOperand(1);
    1461             : 
    1462             :   // Get sign bit into an integer value.
    1463             :   FloatSignAsInt SignAsInt;
    1464          42 :   getSignAsIntValue(SignAsInt, DL, Sign);
    1465             : 
    1466          84 :   EVT IntVT = SignAsInt.IntValue.getValueType();
    1467          42 :   SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
    1468          42 :   SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
    1469          42 :                                 SignMask);
    1470             : 
    1471             :   // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
    1472          42 :   EVT FloatVT = Mag.getValueType();
    1473          42 :   if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
    1474             :       TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
    1475          10 :     SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
    1476          10 :     SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
    1477           5 :     SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
    1478           5 :                                 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
    1479           5 :     return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
    1480             :   }
    1481             : 
    1482             :   // Transform Mag value to integer, and clear the sign bit.
    1483             :   FloatSignAsInt MagAsInt;
    1484          37 :   getSignAsIntValue(MagAsInt, DL, Mag);
    1485          74 :   EVT MagVT = MagAsInt.IntValue.getValueType();
    1486         148 :   SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
    1487          37 :   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
    1488          37 :                                     ClearSignMask);
    1489             : 
    1490             :   // Get the signbit at the right position for MagAsInt.
    1491          37 :   int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
    1492          37 :   if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
    1493          19 :     if (ShiftAmount > 0) {
    1494          19 :       SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
    1495          38 :       SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
    1496           0 :     } else if (ShiftAmount < 0) {
    1497           0 :       SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
    1498           0 :       SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
    1499             :     }
    1500          38 :     SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
    1501          18 :   } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
    1502           0 :     SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
    1503           0 :     if (ShiftAmount > 0) {
    1504           0 :       SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
    1505           0 :       SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
    1506           0 :     } else if (ShiftAmount < 0) {
    1507           0 :       SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
    1508           0 :       SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
    1509             :     }
    1510             :   }
    1511             : 
    1512             :   // Store the part with the modified sign and convert back to float.
    1513          74 :   SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
    1514          37 :   return modifySignAsInt(MagAsInt, DL, CopiedSign);
    1515             : }
    1516             : 
    1517           2 : SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
    1518             :   SDLoc DL(Node);
    1519           2 :   SDValue Value = Node->getOperand(0);
    1520             : 
    1521             :   // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
    1522           2 :   EVT FloatVT = Value.getValueType();
    1523           2 :   if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
    1524           2 :     SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
    1525           4 :     return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
    1526             :   }
    1527             : 
    1528             :   // Transform value to integer, clear the sign bit and transform back.
    1529             :   FloatSignAsInt ValueAsInt;
    1530           0 :   getSignAsIntValue(ValueAsInt, DL, Value);
    1531           0 :   EVT IntVT = ValueAsInt.IntValue.getValueType();
    1532           0 :   SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
    1533           0 :   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
    1534           0 :                                     ClearSignMask);
    1535           0 :   return modifySignAsInt(ValueAsInt, DL, ClearedSign);
    1536             : }
    1537             : 
    1538         155 : void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
    1539             :                                            SmallVectorImpl<SDValue> &Results) {
    1540         155 :   unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
    1541             :   assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
    1542             :           " not tell us which reg is the stack pointer!");
    1543             :   SDLoc dl(Node);
    1544         310 :   EVT VT = Node->getValueType(0);
    1545             :   SDValue Tmp1 = SDValue(Node, 0);
    1546             :   SDValue Tmp2 = SDValue(Node, 1);
    1547         155 :   SDValue Tmp3 = Node->getOperand(2);
    1548         155 :   SDValue Chain = Tmp1.getOperand(0);
    1549             : 
    1550             :   // Chain the dynamic stack allocation so that it doesn't modify the stack
    1551             :   // pointer when other instructions are using the stack.
    1552         155 :   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
    1553             : 
    1554         310 :   SDValue Size  = Tmp2.getOperand(1);
    1555         155 :   SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
    1556         155 :   Chain = SP.getValue(1);
    1557         310 :   unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
    1558             :   unsigned StackAlign =
    1559         310 :       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
    1560         310 :   Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size);       // Value
    1561         155 :   if (Align > StackAlign)
    1562         104 :     Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
    1563         104 :                        DAG.getConstant(-(uint64_t)Align, dl, VT));
    1564         155 :   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
    1565             : 
    1566         465 :   Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
    1567         465 :                             DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
    1568             : 
    1569         155 :   Results.push_back(Tmp1);
    1570         155 :   Results.push_back(Tmp2);
    1571         155 : }
    1572             : 
    1573             : /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
    1574             : /// target.
    1575             : ///
    1576             : /// If the SETCC has been legalized using AND / OR, then the legalized node
    1577             : /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
    1578             : /// will be set to false.
    1579             : ///
    1580             : /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
    1581             : /// then the values of LHS and RHS will be swapped, CC will be set to the
    1582             : /// new condition, and NeedInvert will be set to false.
    1583             : ///
    1584             : /// If the SETCC has been legalized using the inverse condcode, then LHS and
    1585             : /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
    1586             : /// will be set to true. The caller must invert the result of the SETCC with
    1587             : /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
    1588             : /// of a true/false result.
    1589             : ///
    1590             : /// \returns true if the SetCC has been legalized, false if it hasn't.
    1591        2378 : bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
    1592             :                                                  SDValue &RHS, SDValue &CC,
    1593             :                                                  bool &NeedInvert,
    1594             :                                                  const SDLoc &dl) {
    1595             :   MVT OpVT = LHS.getSimpleValueType();
    1596        2378 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
    1597        2378 :   NeedInvert = false;
    1598             :   bool NeedSwap = false;
    1599        4756 :   switch (TLI.getCondCodeAction(CCCode, OpVT)) {
    1600           0 :   default: llvm_unreachable("Unknown condition code action!");
    1601             :   case TargetLowering::Legal:
    1602             :     // Nothing to do.
    1603             :     break;
    1604         370 :   case TargetLowering::Expand: {
    1605         370 :     ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
    1606         370 :     if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
    1607             :       std::swap(LHS, RHS);
    1608         183 :       CC = DAG.getCondCode(InvCC);
    1609             :       return true;
    1610             :     }
    1611             :     // Swapping operands didn't work. Try inverting the condition.
    1612         187 :     InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
    1613         187 :     if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
    1614             :       // If inverting the condition is not enough, try swapping operands
    1615             :       // on top of it.
    1616         142 :       InvCC = ISD::getSetCCSwappedOperands(InvCC);
    1617             :       NeedSwap = true;
    1618             :     }
    1619         187 :     if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
    1620          59 :       CC = DAG.getCondCode(InvCC);
    1621          59 :       NeedInvert = true;
    1622          59 :       if (NeedSwap)
    1623             :         std::swap(LHS, RHS);
    1624             :       return true;
    1625             :     }
    1626             : 
    1627             :     ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
    1628             :     unsigned Opc = 0;
    1629         128 :     switch (CCCode) {
    1630           0 :     default: llvm_unreachable("Don't know how to expand this condition!");
    1631             :     case ISD::SETO:
    1632             :         assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)
    1633             :             && "If SETO is expanded, SETOEQ must be legal!");
    1634             :         CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
    1635          15 :     case ISD::SETUO:
    1636             :         assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)
    1637             :             && "If SETUO is expanded, SETUNE must be legal!");
    1638             :         CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;  break;
    1639             :     case ISD::SETOEQ:
    1640             :     case ISD::SETOGT:
    1641             :     case ISD::SETOGE:
    1642             :     case ISD::SETOLT:
    1643             :     case ISD::SETOLE:
    1644             :     case ISD::SETONE:
    1645             :     case ISD::SETUEQ:
    1646             :     case ISD::SETUNE:
    1647             :     case ISD::SETUGT:
    1648             :     case ISD::SETUGE:
    1649             :     case ISD::SETULT:
    1650             :     case ISD::SETULE:
    1651             :         // If we are floating point, assign and break, otherwise fall through.
    1652         101 :         if (!OpVT.isInteger()) {
    1653             :           // We can use the 4th bit to tell if we are the unordered
    1654             :           // or ordered version of the opcode.
    1655         101 :           CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
    1656         101 :           Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
    1657         101 :           CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
    1658             :           break;
    1659             :         }
    1660             :         // Fallthrough if we are unsigned integer.
    1661             :         LLVM_FALLTHROUGH;
    1662             :     case ISD::SETLE:
    1663             :     case ISD::SETGT:
    1664             :     case ISD::SETGE:
    1665             :     case ISD::SETLT:
    1666             :     case ISD::SETNE:
    1667             :     case ISD::SETEQ:
    1668             :       // If all combinations of inverting the condition and swapping operands
    1669             :       // didn't work then we have no means to expand the condition.
    1670           0 :       llvm_unreachable("Don't know how to expand this condition!");
    1671             :     }
    1672             : 
    1673         128 :     SDValue SetCC1, SetCC2;
    1674         128 :     if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
    1675             :       // If we aren't the ordered or unorder operation,
    1676             :       // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
    1677         101 :       SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
    1678         101 :       SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
    1679             :     } else {
    1680             :       // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
    1681          27 :       SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
    1682          27 :       SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
    1683             :     }
    1684         256 :     LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
    1685         128 :     RHS = SDValue();
    1686         128 :     CC  = SDValue();
    1687             :     return true;
    1688             :   }
    1689             :   }
    1690             :   return false;
    1691             : }
    1692             : 
    1693             : /// Emit a store/load combination to the stack.  This stores
    1694             : /// SrcOp to a stack slot of type SlotVT, truncating it if needed.  It then does
    1695             : /// a load from the stack slot to DestVT, extending it if needed.
    1696             : /// The resultant code need not be legal.
    1697         400 : SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
    1698             :                                                EVT DestVT, const SDLoc &dl) {
    1699             :   // Create the stack frame object.
    1700         800 :   unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
    1701        1200 :       SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
    1702         400 :   SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
    1703             : 
    1704             :   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
    1705         400 :   int SPFI = StackPtrFI->getIndex();
    1706             :   MachinePointerInfo PtrInfo =
    1707         400 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
    1708             : 
    1709         400 :   unsigned SrcSize = SrcOp.getValueSizeInBits();
    1710         400 :   unsigned SlotSize = SlotVT.getSizeInBits();
    1711         400 :   unsigned DestSize = DestVT.getSizeInBits();
    1712         400 :   Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
    1713         800 :   unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
    1714             : 
    1715             :   // Emit a store to the stack slot.  Use a truncstore if the input value is
    1716             :   // later than DestVT.
    1717         400 :   SDValue Store;
    1718             : 
    1719         400 :   if (SrcSize > SlotSize)
    1720           0 :     Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
    1721           0 :                               SlotVT, SrcAlign);
    1722             :   else {
    1723             :     assert(SrcSize == SlotSize && "Invalid store");
    1724         400 :     Store =
    1725        1200 :         DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
    1726             :   }
    1727             : 
    1728             :   // Result is a load from the stack slot.
    1729         400 :   if (SlotSize == DestSize)
    1730         800 :     return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
    1731             : 
    1732             :   assert(SlotSize < DestSize && "Unknown extension!");
    1733           0 :   return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
    1734           0 :                         DestAlign);
    1735             : }
    1736             : 
    1737          18 : SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
    1738             :   SDLoc dl(Node);
    1739             :   // Create a vector sized/aligned stack slot, store the value to element #0,
    1740             :   // then load the whole vector back out.
    1741          36 :   SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
    1742             : 
    1743             :   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
    1744          18 :   int SPFI = StackPtrFI->getIndex();
    1745             : 
    1746          18 :   SDValue Ch = DAG.getTruncStore(
    1747          36 :       DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
    1748          18 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
    1749          72 :       Node->getValueType(0).getVectorElementType());
    1750          18 :   return DAG.getLoad(
    1751             :       Node->getValueType(0), dl, Ch, StackPtr,
    1752          54 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
    1753             : }
    1754             : 
    1755             : static bool
    1756          27 : ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
    1757             :                      const TargetLowering &TLI, SDValue &Res) {
    1758          27 :   unsigned NumElems = Node->getNumOperands();
    1759             :   SDLoc dl(Node);
    1760          54 :   EVT VT = Node->getValueType(0);
    1761             : 
    1762             :   // Try to group the scalars into pairs, shuffle the pairs together, then
    1763             :   // shuffle the pairs of pairs together, etc. until the vector has
    1764             :   // been built. This will work only if all of the necessary shuffle masks
    1765             :   // are legal.
    1766             : 
    1767             :   // We do this in two phases; first to check the legality of the shuffles,
    1768             :   // and next, assuming that all shuffles are legal, to create the new nodes.
    1769         135 :   for (int Phase = 0; Phase < 2; ++Phase) {
    1770          54 :     SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
    1771          54 :                                                               NewIntermedVals;
    1772         742 :     for (unsigned i = 0; i < NumElems; ++i) {
    1773         688 :       SDValue V = Node->getOperand(i);
    1774         344 :       if (V.isUndef())
    1775          16 :         continue;
    1776             : 
    1777             :       SDValue Vec;
    1778         328 :       if (Phase)
    1779         164 :         Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
    1780        1312 :       IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
    1781             :     }
    1782             : 
    1783         206 :     while (IntermedVals.size() > 2) {
    1784          76 :       NewIntermedVals.clear();
    1785         296 :       for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
    1786             :         // This vector and the next vector are shuffled together (simply to
    1787             :         // append the one to the other).
    1788         440 :         SmallVector<int, 16> ShuffleVec(NumElems, -1);
    1789             : 
    1790             :         SmallVector<int, 16> FinalIndices;
    1791         440 :         FinalIndices.reserve(IntermedVals[i].second.size() +
    1792         220 :                              IntermedVals[i+1].second.size());
    1793             : 
    1794             :         int k = 0;
    1795         520 :         for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
    1796             :              ++j, ++k) {
    1797         600 :           ShuffleVec[k] = j;
    1798         600 :           FinalIndices.push_back(IntermedVals[i].second[j]);
    1799             :         }
    1800         520 :         for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
    1801             :              ++j, ++k) {
    1802         600 :           ShuffleVec[k] = NumElems + j;
    1803         600 :           FinalIndices.push_back(IntermedVals[i+1].second[j]);
    1804             :         }
    1805             : 
    1806             :         SDValue Shuffle;
    1807         220 :         if (Phase)
    1808         110 :           Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
    1809             :                                          IntermedVals[i+1].first,
    1810         110 :                                          ShuffleVec);
    1811         220 :         else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
    1812             :           return false;
    1813         220 :         NewIntermedVals.push_back(
    1814         220 :             std::make_pair(Shuffle, std::move(FinalIndices)));
    1815             :       }
    1816             : 
    1817             :       // If we had an odd number of defined values, then append the last
    1818             :       // element to the array of new vectors.
    1819          76 :       if ((IntermedVals.size() & 1) != 0)
    1820           0 :         NewIntermedVals.push_back(IntermedVals.back());
    1821             : 
    1822          76 :       IntermedVals.swap(NewIntermedVals);
    1823             :     }
    1824             : 
    1825             :     assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
    1826             :            "Invalid number of intermediate vectors");
    1827          54 :     SDValue Vec1 = IntermedVals[0].first;
    1828          54 :     SDValue Vec2;
    1829          54 :     if (IntermedVals.size() > 1)
    1830          54 :       Vec2 = IntermedVals[1].first;
    1831           0 :     else if (Phase)
    1832           0 :       Vec2 = DAG.getUNDEF(VT);
    1833             : 
    1834         108 :     SmallVector<int, 16> ShuffleVec(NumElems, -1);
    1835         218 :     for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
    1836         492 :       ShuffleVec[IntermedVals[0].second[i]] = i;
    1837         218 :     for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
    1838         492 :       ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
    1839             : 
    1840          54 :     if (Phase)
    1841          27 :       Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
    1842          54 :     else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
    1843             :       return false;
    1844             :   }
    1845             : 
    1846             :   return true;
    1847             : }
    1848             : 
    1849             : /// Expand a BUILD_VECTOR node on targets that don't
    1850             : /// support the operation, but do support the resultant vector type.
    1851       21842 : SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
    1852       21842 :   unsigned NumElems = Node->getNumOperands();
    1853             :   SDValue Value1, Value2;
    1854             :   SDLoc dl(Node);
    1855       43684 :   EVT VT = Node->getValueType(0);
    1856       21842 :   EVT OpVT = Node->getOperand(0).getValueType();
    1857       21842 :   EVT EltVT = VT.getVectorElementType();
    1858             : 
    1859             :   // If the only non-undef value is the low element, turn this into a
    1860             :   // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
    1861             :   bool isOnlyLowElement = true;
    1862             :   bool MoreThanTwoValues = false;
    1863             :   bool isConstant = true;
    1864      539794 :   for (unsigned i = 0; i < NumElems; ++i) {
    1865      517952 :     SDValue V = Node->getOperand(i);
    1866      258976 :     if (V.isUndef())
    1867             :       continue;
    1868      244651 :     if (i > 0)
    1869             :       isOnlyLowElement = false;
    1870             :     if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
    1871             :       isConstant = false;
    1872             : 
    1873      244651 :     if (!Value1.getNode()) {
    1874             :       Value1 = V;
    1875      222809 :     } else if (!Value2.getNode()) {
    1876             :       if (V != Value1)
    1877             :         Value2 = V;
    1878             :     } else if (V != Value1 && V != Value2) {
    1879             :       MoreThanTwoValues = true;
    1880             :     }
    1881             :   }
    1882             : 
    1883       21842 :   if (!Value1.getNode())
    1884           0 :     return DAG.getUNDEF(VT);
    1885             : 
    1886       21842 :   if (isOnlyLowElement)
    1887          45 :     return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
    1888             : 
    1889             :   // If all elements are constants, create a load from the constant pool.
    1890       21827 :   if (isConstant) {
    1891             :     SmallVector<Constant*, 16> CV;
    1892      519910 :     for (unsigned i = 0, e = NumElems; i != e; ++i) {
    1893             :       if (ConstantFPSDNode *V =
    1894      249960 :           dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
    1895        7908 :         CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
    1896             :       } else if (ConstantSDNode *V =
    1897             :                  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
    1898      227990 :         if (OpVT==EltVT)
    1899      224578 :           CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
    1900             :         else {
    1901             :           // If OpVT and EltVT don't match, EltVT is not legal and the
    1902             :           // element values have been promoted/truncated earlier.  Undo this;
    1903             :           // we don't want a v16i8 to become a v16i32 for example.
    1904        3412 :           const ConstantInt *CI = V->getConstantIntValue();
    1905        3412 :           CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
    1906             :                                         CI->getZExtValue()));
    1907             :         }
    1908             :       } else {
    1909             :         assert(Node->getOperand(i).isUndef());
    1910       14062 :         Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
    1911       14062 :         CV.push_back(UndefValue::get(OpNTy));
    1912             :       }
    1913             :     }
    1914       19990 :     Constant *CP = ConstantVector::get(CV);
    1915             :     SDValue CPIdx =
    1916       39980 :         DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
    1917       19990 :     unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
    1918       19990 :     return DAG.getLoad(
    1919       19990 :         VT, dl, DAG.getEntryNode(), CPIdx,
    1920             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
    1921       39980 :         Alignment);
    1922             :   }
    1923             : 
    1924        1837 :   SmallSet<SDValue, 16> DefinedValues;
    1925       19805 :   for (unsigned i = 0; i < NumElems; ++i) {
    1926       26952 :     if (Node->getOperand(i).isUndef())
    1927         246 :       continue;
    1928        8738 :     DefinedValues.insert(Node->getOperand(i));
    1929             :   }
    1930             : 
    1931        3674 :   if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
    1932        1773 :     if (!MoreThanTwoValues) {
    1933        3492 :       SmallVector<int, 8> ShuffleVec(NumElems, -1);
    1934       18990 :       for (unsigned i = 0; i < NumElems; ++i) {
    1935       17244 :         SDValue V = Node->getOperand(i);
    1936        8622 :         if (V.isUndef())
    1937             :           continue;
    1938        9540 :         ShuffleVec[i] = V == Value1 ? 0 : NumElems;
    1939             :       }
    1940        3492 :       if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
    1941             :         // Get the splatted value into the low element of a vector register.
    1942        3318 :         SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
    1943        1659 :         SDValue Vec2;
    1944        1659 :         if (Value2.getNode())
    1945        2136 :           Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
    1946             :         else
    1947         591 :           Vec2 = DAG.getUNDEF(VT);
    1948             : 
    1949             :         // Return shuffle(LowValVec, undef, <0,0,0,0>)
    1950        3318 :         return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
    1951             :       }
    1952             :     } else {
    1953          27 :       SDValue Res;
    1954          27 :       if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
    1955          27 :         return Res;
    1956             :     }
    1957             :   }
    1958             : 
    1959             :   // Otherwise, we can't handle this case efficiently.
    1960         151 :   return ExpandVectorBuildThroughStack(Node);
    1961             : }
    1962             : 
    1963             : // Expand a node into a call to a libcall.  If the result value
    1964             : // does not fit into a register, return the lo part and set the hi part to the
    1965             : // by-reg argument.  If it does fit into a single register, return the result
    1966             : // and leave the Hi part unset.
    1967        2649 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
    1968             :                                             bool isSigned) {
    1969             :   TargetLowering::ArgListTy Args;
    1970             :   TargetLowering::ArgListEntry Entry;
    1971        6502 :   for (const SDValue &Op : Node->op_values()) {
    1972        7706 :     EVT ArgVT = Op.getValueType();
    1973        3853 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    1974        3853 :     Entry.Node = Op;
    1975        3853 :     Entry.Ty = ArgTy;
    1976        3853 :     Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
    1977        3853 :     Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
    1978        3853 :     Args.push_back(Entry);
    1979             :   }
    1980        5298 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    1981        5298 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    1982             : 
    1983        5298 :   EVT RetVT = Node->getValueType(0);
    1984        2649 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    1985             : 
    1986             :   // By default, the input chain to this libcall is the entry node of the
    1987             :   // function. If the libcall is going to be emitted as a tail call then
    1988             :   // TLI.isUsedByReturnOnly will change it to the right chain if the return
    1989             :   // node which is being folded has a non-entry input chain.
    1990        2649 :   SDValue InChain = DAG.getEntryNode();
    1991             : 
    1992             :   // isTailCall may be true since the callee does not reference caller stack
    1993             :   // frame. Check if it's in the right position and that the return types match.
    1994        2649 :   SDValue TCChain = InChain;
    1995        2649 :   const Function &F = DAG.getMachineFunction().getFunction();
    1996             :   bool isTailCall =
    1997        2997 :       TLI.isInTailCallPosition(DAG, Node, TCChain) &&
    1998           6 :       (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
    1999             :   if (isTailCall)
    2000         342 :     InChain = TCChain;
    2001             : 
    2002        5298 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2003        2649 :   bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
    2004        2649 :   CLI.setDebugLoc(SDLoc(Node))
    2005             :       .setChain(InChain)
    2006        2649 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2007        2649 :                     std::move(Args))
    2008             :       .setTailCall(isTailCall)
    2009             :       .setSExtResult(signExtend)
    2010        2649 :       .setZExtResult(!signExtend)
    2011             :       .setIsPostTypeLegalization(true);
    2012             : 
    2013        2649 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2014             : 
    2015        2649 :   if (!CallInfo.second.getNode()) {
    2016             :     LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump());
    2017             :     // It's a tailcall, return the chain (which is the DAG root).
    2018         310 :     return DAG.getRoot();
    2019             :   }
    2020             : 
    2021             :   LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump());
    2022        2339 :   return CallInfo.first;
    2023             : }
    2024             : 
    2025             : /// Generate a libcall taking the given operands as arguments
    2026             : /// and returning a result of type RetVT.
    2027           8 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
    2028             :                                             const SDValue *Ops, unsigned NumOps,
    2029             :                                             bool isSigned, const SDLoc &dl) {
    2030             :   TargetLowering::ArgListTy Args;
    2031           8 :   Args.reserve(NumOps);
    2032             : 
    2033             :   TargetLowering::ArgListEntry Entry;
    2034          72 :   for (unsigned i = 0; i != NumOps; ++i) {
    2035          32 :     Entry.Node = Ops[i];
    2036          64 :     Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
    2037          32 :     Entry.IsSExt = isSigned;
    2038          32 :     Entry.IsZExt = !isSigned;
    2039          32 :     Args.push_back(Entry);
    2040             :   }
    2041          16 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2042          16 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2043             : 
    2044           8 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2045             : 
    2046          16 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2047             :   CLI.setDebugLoc(dl)
    2048           8 :       .setChain(DAG.getEntryNode())
    2049           8 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2050           8 :                     std::move(Args))
    2051             :       .setSExtResult(isSigned)
    2052           8 :       .setZExtResult(!isSigned)
    2053             :       .setIsPostTypeLegalization(true);
    2054             : 
    2055           8 :   std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2056             : 
    2057          16 :   return CallInfo.first;
    2058             : }
    2059             : 
    2060             : // Expand a node into a call to a libcall. Similar to
    2061             : // ExpandLibCall except that the first operand is the in-chain.
    2062             : std::pair<SDValue, SDValue>
    2063         325 : SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
    2064             :                                          SDNode *Node,
    2065             :                                          bool isSigned) {
    2066         325 :   SDValue InChain = Node->getOperand(0);
    2067             : 
    2068             :   TargetLowering::ArgListTy Args;
    2069             :   TargetLowering::ArgListEntry Entry;
    2070        1988 :   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
    2071        2007 :     EVT ArgVT = Node->getOperand(i).getValueType();
    2072         669 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2073        1338 :     Entry.Node = Node->getOperand(i);
    2074         669 :     Entry.Ty = ArgTy;
    2075         669 :     Entry.IsSExt = isSigned;
    2076         669 :     Entry.IsZExt = !isSigned;
    2077         669 :     Args.push_back(Entry);
    2078             :   }
    2079         650 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2080         650 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2081             : 
    2082         650 :   Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
    2083             : 
    2084         650 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2085         325 :   CLI.setDebugLoc(SDLoc(Node))
    2086             :       .setChain(InChain)
    2087         325 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2088         325 :                     std::move(Args))
    2089             :       .setSExtResult(isSigned)
    2090         325 :       .setZExtResult(!isSigned);
    2091             : 
    2092         325 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2093             : 
    2094         325 :   return CallInfo;
    2095             : }
    2096             : 
    2097        1710 : SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
    2098             :                                               RTLIB::Libcall Call_F32,
    2099             :                                               RTLIB::Libcall Call_F64,
    2100             :                                               RTLIB::Libcall Call_F80,
    2101             :                                               RTLIB::Libcall Call_F128,
    2102             :                                               RTLIB::Libcall Call_PPCF128) {
    2103        1710 :   if (Node->isStrictFPOpcode())
    2104          68 :     Node = DAG.mutateStrictFPToFP(Node);
    2105             : 
    2106             :   RTLIB::Libcall LC;
    2107        1710 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2108           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2109             :   case MVT::f32: LC = Call_F32; break;
    2110         724 :   case MVT::f64: LC = Call_F64; break;
    2111          67 :   case MVT::f80: LC = Call_F80; break;
    2112          46 :   case MVT::f128: LC = Call_F128; break;
    2113           0 :   case MVT::ppcf128: LC = Call_PPCF128; break;
    2114             :   }
    2115        1710 :   return ExpandLibCall(LC, Node, false);
    2116             : }
    2117             : 
    2118         116 : SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
    2119             :                                                RTLIB::Libcall Call_I8,
    2120             :                                                RTLIB::Libcall Call_I16,
    2121             :                                                RTLIB::Libcall Call_I32,
    2122             :                                                RTLIB::Libcall Call_I64,
    2123             :                                                RTLIB::Libcall Call_I128) {
    2124             :   RTLIB::Libcall LC;
    2125         116 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2126           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2127             :   case MVT::i8:   LC = Call_I8; break;
    2128          13 :   case MVT::i16:  LC = Call_I16; break;
    2129         103 :   case MVT::i32:  LC = Call_I32; break;
    2130           0 :   case MVT::i64:  LC = Call_I64; break;
    2131           0 :   case MVT::i128: LC = Call_I128; break;
    2132             :   }
    2133         116 :   return ExpandLibCall(LC, Node, isSigned);
    2134             : }
    2135             : 
    2136             : /// Issue libcalls to __{u}divmod to compute div / rem pairs.
    2137             : void
    2138          14 : SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
    2139             :                                           SmallVectorImpl<SDValue> &Results) {
    2140          14 :   unsigned Opcode = Node->getOpcode();
    2141          14 :   bool isSigned = Opcode == ISD::SDIVREM;
    2142             : 
    2143             :   RTLIB::Libcall LC;
    2144          14 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2145           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2146           0 :   case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
    2147           0 :   case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
    2148          14 :   case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
    2149           0 :   case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
    2150           0 :   case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
    2151             :   }
    2152             : 
    2153             :   // The input chain to this libcall is the entry node of the function.
    2154             :   // Legalizing the call will automatically add the previous call to the
    2155             :   // dependence.
    2156          14 :   SDValue InChain = DAG.getEntryNode();
    2157             : 
    2158          14 :   EVT RetVT = Node->getValueType(0);
    2159          14 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2160             : 
    2161             :   TargetLowering::ArgListTy Args;
    2162             :   TargetLowering::ArgListEntry Entry;
    2163          42 :   for (const SDValue &Op : Node->op_values()) {
    2164          56 :     EVT ArgVT = Op.getValueType();
    2165          28 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2166          28 :     Entry.Node = Op;
    2167          28 :     Entry.Ty = ArgTy;
    2168          28 :     Entry.IsSExt = isSigned;
    2169          28 :     Entry.IsZExt = !isSigned;
    2170          28 :     Args.push_back(Entry);
    2171             :   }
    2172             : 
    2173             :   // Also pass the return address of the remainder.
    2174          14 :   SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
    2175          14 :   Entry.Node = FIPtr;
    2176          14 :   Entry.Ty = RetTy->getPointerTo();
    2177          14 :   Entry.IsSExt = isSigned;
    2178          14 :   Entry.IsZExt = !isSigned;
    2179          14 :   Args.push_back(Entry);
    2180             : 
    2181          28 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2182          28 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2183             : 
    2184             :   SDLoc dl(Node);
    2185          28 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2186             :   CLI.setDebugLoc(dl)
    2187             :       .setChain(InChain)
    2188          14 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2189          14 :                     std::move(Args))
    2190             :       .setSExtResult(isSigned)
    2191             :       .setZExtResult(!isSigned);
    2192             : 
    2193          14 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2194             : 
    2195             :   // Remainder is loaded back from the stack frame.
    2196             :   SDValue Rem =
    2197          28 :       DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
    2198          14 :   Results.push_back(CallInfo.first);
    2199          14 :   Results.push_back(Rem);
    2200          14 : }
    2201             : 
    2202             : /// Return true if sincos libcall is available.
    2203             : static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
    2204             :   RTLIB::Libcall LC;
    2205         269 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2206           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2207             :   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
    2208          94 :   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
    2209          19 :   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
    2210          14 :   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
    2211           0 :   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
    2212             :   }
    2213             :   return TLI.getLibcallName(LC) != nullptr;
    2214             : }
    2215             : 
    2216             : /// Only issue sincos libcall if both sin and cos are needed.
    2217             : static bool useSinCos(SDNode *Node) {
    2218             :   unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
    2219         140 :     ? ISD::FCOS : ISD::FSIN;
    2220             : 
    2221         140 :   SDValue Op0 = Node->getOperand(0);
    2222         140 :   for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
    2223         202 :        UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
    2224             :     SDNode *User = *UI;
    2225         140 :     if (User == Node)
    2226             :       continue;
    2227             :     // The other user might have been turned into sincos already.
    2228         156 :     if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
    2229             :       return true;
    2230             :   }
    2231             :   return false;
    2232             : }
    2233             : 
    2234             : /// Issue libcalls to sincos to compute sin / cos pairs.
    2235             : void
    2236          30 : SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
    2237             :                                           SmallVectorImpl<SDValue> &Results) {
    2238             :   RTLIB::Libcall LC;
    2239          30 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2240           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2241             :   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
    2242          11 :   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
    2243           5 :   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
    2244           3 :   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
    2245           0 :   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
    2246             :   }
    2247             : 
    2248             :   // The input chain to this libcall is the entry node of the function.
    2249             :   // Legalizing the call will automatically add the previous call to the
    2250             :   // dependence.
    2251          30 :   SDValue InChain = DAG.getEntryNode();
    2252             : 
    2253          30 :   EVT RetVT = Node->getValueType(0);
    2254          30 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2255             : 
    2256             :   TargetLowering::ArgListTy Args;
    2257             :   TargetLowering::ArgListEntry Entry;
    2258             : 
    2259             :   // Pass the argument.
    2260          30 :   Entry.Node = Node->getOperand(0);
    2261          30 :   Entry.Ty = RetTy;
    2262             :   Entry.IsSExt = false;
    2263             :   Entry.IsZExt = false;
    2264          30 :   Args.push_back(Entry);
    2265             : 
    2266             :   // Pass the return address of sin.
    2267          30 :   SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
    2268          30 :   Entry.Node = SinPtr;
    2269          30 :   Entry.Ty = RetTy->getPointerTo();
    2270          30 :   Entry.IsSExt = false;
    2271          30 :   Entry.IsZExt = false;
    2272          30 :   Args.push_back(Entry);
    2273             : 
    2274             :   // Also pass the return address of the cos.
    2275          30 :   SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
    2276          30 :   Entry.Node = CosPtr;
    2277          30 :   Entry.Ty = RetTy->getPointerTo();
    2278          30 :   Entry.IsSExt = false;
    2279          30 :   Entry.IsZExt = false;
    2280          30 :   Args.push_back(Entry);
    2281             : 
    2282          60 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2283          60 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2284             : 
    2285             :   SDLoc dl(Node);
    2286          60 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2287          30 :   CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
    2288          60 :       TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
    2289          30 :       std::move(Args));
    2290             : 
    2291          30 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2292             : 
    2293          60 :   Results.push_back(
    2294          90 :       DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
    2295          30 :   Results.push_back(
    2296          90 :       DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
    2297          30 : }
    2298             : 
    2299             : /// This function is responsible for legalizing a
    2300             : /// INT_TO_FP operation of the specified operand when the target requests that
    2301             : /// we expand it.  At this point, we know that the result and operand types are
    2302             : /// legal for the target.
    2303         136 : SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
    2304             :                                                    EVT DestVT,
    2305             :                                                    const SDLoc &dl) {
    2306             :   // TODO: Should any fast-math-flags be set for the created nodes?
    2307             :   LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
    2308          51 :   if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
    2309             :     LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
    2310             :                          "expansion\n");
    2311             : 
    2312             :     // Get the stack frame index of a 8 byte buffer.
    2313         100 :     SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
    2314             : 
    2315             :     // word offset constant for Hi/Lo address computation
    2316          50 :     SDValue WordOff = DAG.getConstant(sizeof(int), dl,
    2317          50 :                                       StackSlot.getValueType());
    2318             :     // set up Hi and Lo (into buffer) address based on endian
    2319          50 :     SDValue Hi = StackSlot;
    2320          50 :     SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
    2321          50 :                              StackSlot, WordOff);
    2322         100 :     if (DAG.getDataLayout().isLittleEndian())
    2323             :       std::swap(Hi, Lo);
    2324             : 
    2325             :     // if signed map to unsigned space
    2326          50 :     SDValue Op0Mapped;
    2327          50 :     if (isSigned) {
    2328             :       // constant used to invert sign bit (signed to unsigned mapping)
    2329           8 :       SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
    2330           8 :       Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
    2331             :     } else {
    2332          46 :       Op0Mapped = Op0;
    2333             :     }
    2334             :     // store the lo of the constructed double - based on integer input
    2335          50 :     SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
    2336          50 :                                   MachinePointerInfo());
    2337             :     // initial hi portion of constructed double
    2338         100 :     SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
    2339             :     // store the hi of the constructed double - biased exponent
    2340             :     SDValue Store2 =
    2341         100 :         DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
    2342             :     // load the constructed double
    2343             :     SDValue Load =
    2344         100 :         DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
    2345             :     // FP constant to bias correct the final result
    2346          50 :     SDValue Bias = DAG.getConstantFP(isSigned ?
    2347             :                                      BitsToDouble(0x4330000080000000ULL) :
    2348             :                                      BitsToDouble(0x4330000000000000ULL),
    2349          50 :                                      dl, MVT::f64);
    2350             :     // subtract the bias
    2351         100 :     SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
    2352             :     // final result
    2353             :     SDValue Result;
    2354             :     // handle final rounding
    2355             :     if (DestVT == MVT::f64) {
    2356             :       // do nothing
    2357          10 :       Result = Sub;
    2358          40 :     } else if (DestVT.bitsLT(MVT::f64)) {
    2359          72 :       Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
    2360          72 :                            DAG.getIntPtrConstant(0, dl));
    2361           4 :     } else if (DestVT.bitsGT(MVT::f64)) {
    2362           8 :       Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
    2363             :     }
    2364          50 :     return Result;
    2365             :   }
    2366             :   assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
    2367             :   // Code below here assumes !isSigned without checking again.
    2368             : 
    2369             :   // Implementation of unsigned i64 to f64 following the algorithm in
    2370             :   // __floatundidf in compiler_rt. This implementation has the advantage
    2371             :   // of performing rounding correctly, both in the default rounding mode
    2372             :   // and in all alternate rounding modes.
    2373             :   // TODO: Generalize this for use with other types.
    2374             :   if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
    2375             :     LLVM_DEBUG(dbgs() << "Converting unsigned i64 to f64\n");
    2376             :     SDValue TwoP52 =
    2377           8 :       DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
    2378             :     SDValue TwoP84PlusTwoP52 =
    2379           4 :       DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
    2380           4 :                         MVT::f64);
    2381             :     SDValue TwoP84 =
    2382           8 :       DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
    2383             : 
    2384           8 :     SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
    2385           4 :     SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
    2386           8 :                              DAG.getConstant(32, dl, MVT::i64));
    2387           8 :     SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
    2388           8 :     SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
    2389           8 :     SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
    2390           8 :     SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
    2391           4 :     SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
    2392           4 :                                 TwoP84PlusTwoP52);
    2393           8 :     return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
    2394             :   }
    2395             : 
    2396             :   // TODO: Generalize this for use with other types.
    2397             :   if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
    2398             :     LLVM_DEBUG(dbgs() << "Converting unsigned i64 to f32\n");
    2399             :     // For unsigned conversions, convert them to signed conversions using the
    2400             :     // algorithm from the x86_64 __floatundidf in compiler_rt.
    2401          80 :     if (!isSigned) {
    2402         160 :       SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
    2403             : 
    2404          80 :       SDValue ShiftConst = DAG.getConstant(
    2405         160 :           1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
    2406         160 :       SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
    2407         160 :       SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
    2408         160 :       SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
    2409         160 :       SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
    2410             : 
    2411         160 :       SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
    2412         160 :       SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
    2413             : 
    2414             :       // TODO: This really should be implemented using a branch rather than a
    2415             :       // select.  We happen to get lucky and machinesink does the right
    2416             :       // thing most of the time.  This would be a good candidate for a
    2417             :       //pseudo-op, or, even better, for whole-function isel.
    2418          80 :       SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
    2419         160 :         Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
    2420         160 :       return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
    2421             :     }
    2422             : 
    2423             :     // Otherwise, implement the fully general conversion.
    2424             : 
    2425           0 :     SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
    2426           0 :          DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
    2427           0 :     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
    2428           0 :          DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
    2429           0 :     SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
    2430           0 :          DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
    2431           0 :     SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
    2432             :                               DAG.getConstant(UINT64_C(0), dl, MVT::i64),
    2433           0 :                               ISD::SETNE);
    2434           0 :     SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
    2435           0 :     SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
    2436             :                               DAG.getConstant(UINT64_C(0x0020000000000000), dl,
    2437             :                                               MVT::i64),
    2438           0 :                               ISD::SETUGE);
    2439           0 :     SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
    2440           0 :     EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
    2441             : 
    2442           0 :     SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
    2443           0 :                              DAG.getConstant(32, dl, SHVT));
    2444           0 :     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
    2445           0 :     SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
    2446             :     SDValue TwoP32 =
    2447           0 :       DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
    2448           0 :                         MVT::f64);
    2449           0 :     SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
    2450           0 :     SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
    2451           0 :     SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
    2452           0 :     SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
    2453           0 :     return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
    2454           0 :                        DAG.getIntPtrConstant(0, dl));
    2455             :   }
    2456             : 
    2457           4 :   SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
    2458             : 
    2459           2 :   SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
    2460             :                                  Op0,
    2461             :                                  DAG.getConstant(0, dl, Op0.getValueType()),
    2462           4 :                                  ISD::SETLT);
    2463           2 :   SDValue Zero = DAG.getIntPtrConstant(0, dl),
    2464           2 :           Four = DAG.getIntPtrConstant(4, dl);
    2465           2 :   SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
    2466           4 :                                     SignSet, Four, Zero);
    2467             : 
    2468             :   // If the sign bit of the integer is set, the large number will be treated
    2469             :   // as a negative number.  To counteract this, the dynamic code adds an
    2470             :   // offset depending on the data type.
    2471             :   uint64_t FF;
    2472           2 :   switch (Op0.getSimpleValueType().SimpleTy) {
    2473           0 :   default: llvm_unreachable("Unsupported integer type!");
    2474             :   case MVT::i8 : FF = 0x43800000ULL; break;  // 2^8  (as a float)
    2475           0 :   case MVT::i16: FF = 0x47800000ULL; break;  // 2^16 (as a float)
    2476           1 :   case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
    2477           1 :   case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
    2478             :   }
    2479           4 :   if (DAG.getDataLayout().isLittleEndian())
    2480           0 :     FF <<= 32;
    2481           2 :   Constant *FudgeFactor = ConstantInt::get(
    2482           4 :                                        Type::getInt64Ty(*DAG.getContext()), FF);
    2483             : 
    2484             :   SDValue CPIdx =
    2485           4 :       DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
    2486           2 :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
    2487           4 :   CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
    2488           4 :   Alignment = std::min(Alignment, 4u);
    2489           2 :   SDValue FudgeInReg;
    2490             :   if (DestVT == MVT::f32)
    2491           2 :     FudgeInReg = DAG.getLoad(
    2492           1 :         MVT::f32, dl, DAG.getEntryNode(), CPIdx,
    2493             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
    2494           2 :         Alignment);
    2495             :   else {
    2496           1 :     SDValue Load = DAG.getExtLoad(
    2497           1 :         ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
    2498             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
    2499           2 :         Alignment);
    2500           2 :     HandleSDNode Handle(Load);
    2501           1 :     LegalizeOp(Load.getNode());
    2502           1 :     FudgeInReg = Handle.getValue();
    2503             :   }
    2504             : 
    2505           4 :   return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
    2506             : }
    2507             : 
    2508             : /// This function is responsible for legalizing a
    2509             : /// *INT_TO_FP operation of the specified operand when the target requests that
    2510             : /// we promote it.  At this point, we know that the result and operand types are
    2511             : /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
    2512             : /// operation that takes a larger input.
    2513          93 : SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
    2514             :                                                     bool isSigned,
    2515             :                                                     const SDLoc &dl) {
    2516             :   // First step, figure out the appropriate *INT_TO_FP operation to use.
    2517          93 :   EVT NewInTy = LegalOp.getValueType();
    2518             : 
    2519             :   unsigned OpToUse = 0;
    2520             : 
    2521             :   // Scan for the appropriate larger type to use.
    2522             :   while (true) {
    2523          99 :     NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
    2524             :     assert(NewInTy.isInteger() && "Ran out of possibilities!");
    2525             : 
    2526             :     // If the target supports SINT_TO_FP of this type, use it.
    2527          99 :     if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
    2528             :       OpToUse = ISD::SINT_TO_FP;
    2529             :       break;
    2530             :     }
    2531           6 :     if (isSigned) continue;
    2532             : 
    2533             :     // If the target supports UINT_TO_FP of this type, use it.
    2534             :     if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
    2535             :       OpToUse = ISD::UINT_TO_FP;
    2536             :       break;
    2537             :     }
    2538             : 
    2539             :     // Otherwise, try a larger type.
    2540             :   }
    2541             : 
    2542             :   // Okay, we found the operation and type to use.  Zero extend our input to the
    2543             :   // desired type then run the operation on it.
    2544          93 :   return DAG.getNode(OpToUse, dl, DestVT,
    2545             :                      DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
    2546          93 :                                  dl, NewInTy, LegalOp));
    2547             : }
    2548             : 
    2549             : /// This function is responsible for legalizing a
    2550             : /// FP_TO_*INT operation of the specified operand when the target requests that
    2551             : /// we promote it.  At this point, we know that the result and operand types are
    2552             : /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
    2553             : /// operation that returns a larger result.
    2554         142 : SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
    2555             :                                                     bool isSigned,
    2556             :                                                     const SDLoc &dl) {
    2557             :   // First step, figure out the appropriate FP_TO*INT operation to use.
    2558         142 :   EVT NewOutTy = DestVT;
    2559             : 
    2560             :   unsigned OpToUse = 0;
    2561             : 
    2562             :   // Scan for the appropriate larger type to use.
    2563             :   while (true) {
    2564         178 :     NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
    2565             :     assert(NewOutTy.isInteger() && "Ran out of possibilities!");
    2566             : 
    2567             :     // A larger signed type can hold all unsigned values of the requested type,
    2568             :     // so using FP_TO_SINT is valid
    2569         178 :     if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
    2570             :       OpToUse = ISD::FP_TO_SINT;
    2571             :       break;
    2572             :     }
    2573             : 
    2574             :     // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
    2575          36 :     if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
    2576             :       OpToUse = ISD::FP_TO_UINT;
    2577             :       break;
    2578             :     }
    2579             : 
    2580             :     // Otherwise, try a larger type.
    2581             :   }
    2582             : 
    2583             :   // Okay, we found the operation and type to use.
    2584         284 :   SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
    2585             : 
    2586             :   // Truncate the result of the extended FP_TO_*INT operation to the desired
    2587             :   // size.
    2588         284 :   return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
    2589             : }
    2590             : 
    2591             : /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
    2592          92 : SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
    2593         184 :   EVT VT = Op.getValueType();
    2594         184 :   EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2595             :   unsigned Sz = VT.getScalarSizeInBits();
    2596             : 
    2597          92 :   SDValue Tmp, Tmp2, Tmp3;
    2598             : 
    2599             :   // If we can, perform BSWAP first and then the mask+swap the i4, then i2
    2600             :   // and finally the i1 pairs.
    2601             :   // TODO: We can easily support i4/i2 legal types if any target ever does.
    2602          92 :   if (Sz >= 8 && isPowerOf2_32(Sz)) {
    2603             :     // Create the masks - repeating the pattern every byte.
    2604             :     APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
    2605             :     APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
    2606         756 :     for (unsigned J = 0; J != Sz; J += 8) {
    2607        1328 :       MaskHi4 = MaskHi4 | (0xF0ull << J);
    2608        1328 :       MaskLo4 = MaskLo4 | (0x0Full << J);
    2609        1328 :       MaskHi2 = MaskHi2 | (0xCCull << J);
    2610        1328 :       MaskLo2 = MaskLo2 | (0x33ull << J);
    2611        1328 :       MaskHi1 = MaskHi1 | (0xAAull << J);
    2612        1328 :       MaskLo1 = MaskLo1 | (0x55ull << J);
    2613             :     }
    2614             : 
    2615             :     // BSWAP if the type is wider than a single byte.
    2616         144 :     Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
    2617             : 
    2618             :     // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
    2619         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
    2620         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
    2621         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
    2622         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
    2623         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2624             : 
    2625             :     // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
    2626         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
    2627         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
    2628         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
    2629         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
    2630         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2631             : 
    2632             :     // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
    2633         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
    2634         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
    2635         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
    2636         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
    2637         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2638          92 :     return Tmp;
    2639             :   }
    2640             : 
    2641           0 :   Tmp = DAG.getConstant(0, dl, VT);
    2642           0 :   for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
    2643           0 :     if (I < J)
    2644           0 :       Tmp2 =
    2645           0 :           DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
    2646             :     else
    2647           0 :       Tmp2 =
    2648           0 :           DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
    2649             : 
    2650             :     APInt Shift(Sz, 1);
    2651           0 :     Shift <<= J;
    2652           0 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
    2653           0 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
    2654             :   }
    2655             : 
    2656           0 :   return Tmp;
    2657             : }
    2658             : 
    2659             : /// Open code the operations for BSWAP of the specified operation.
    2660         133 : SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
    2661         266 :   EVT VT = Op.getValueType();
    2662         266 :   EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2663         133 :   SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
    2664         266 :   switch (VT.getSimpleVT().getScalarType().SimpleTy) {
    2665           0 :   default: llvm_unreachable("Unhandled Expand type in BSWAP!");
    2666          69 :   case MVT::i16:
    2667         138 :     Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2668         138 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2669         138 :     return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
    2670          25 :   case MVT::i32:
    2671          50 :     Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2672          50 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2673          50 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2674          50 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2675          50 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
    2676          50 :                        DAG.getConstant(0xFF0000, dl, VT));
    2677          50 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
    2678          50 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    2679          50 :     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    2680          50 :     return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
    2681          39 :   case MVT::i64:
    2682          78 :     Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    2683          78 :     Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
    2684          78 :     Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2685          78 :     Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2686          78 :     Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2687          78 :     Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2688          78 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
    2689          78 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    2690          78 :     Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
    2691          78 :                        DAG.getConstant(255ULL<<48, dl, VT));
    2692          78 :     Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
    2693          78 :                        DAG.getConstant(255ULL<<40, dl, VT));
    2694          78 :     Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
    2695          78 :                        DAG.getConstant(255ULL<<32, dl, VT));
    2696          78 :     Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
    2697          78 :                        DAG.getConstant(255ULL<<24, dl, VT));
    2698          78 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
    2699          78 :                        DAG.getConstant(255ULL<<16, dl, VT));
    2700          78 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
    2701          78 :                        DAG.getConstant(255ULL<<8 , dl, VT));
    2702          78 :     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
    2703          78 :     Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
    2704          78 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    2705          78 :     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    2706          78 :     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
    2707          78 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
    2708          78 :     return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
    2709             :   }
    2710             : }
    2711             : 
    2712             : /// Expand the specified bitcount instruction into operations.
    2713         369 : SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
    2714             :                                              const SDLoc &dl) {
    2715         369 :   switch (Opc) {
    2716           0 :   default: llvm_unreachable("Cannot expand this yet!");
    2717          68 :   case ISD::CTPOP: {
    2718          68 :     EVT VT = Op.getValueType();
    2719         136 :     EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2720          68 :     unsigned Len = VT.getSizeInBits();
    2721             : 
    2722             :     assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
    2723             :            "CTPOP not implemented for this type.");
    2724             : 
    2725             :     // This is the "best" algorithm from
    2726             :     // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
    2727             : 
    2728         272 :     SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
    2729          68 :                                      dl, VT);
    2730         272 :     SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
    2731          68 :                                      dl, VT);
    2732         272 :     SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
    2733          68 :                                      dl, VT);
    2734         272 :     SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
    2735          68 :                                      dl, VT);
    2736             : 
    2737             :     // v = v - ((v >> 1) & 0x55555555...)
    2738         136 :     Op = DAG.getNode(ISD::SUB, dl, VT, Op,
    2739             :                      DAG.getNode(ISD::AND, dl, VT,
    2740             :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2741             :                                              DAG.getConstant(1, dl, ShVT)),
    2742         136 :                                  Mask55));
    2743             :     // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
    2744         136 :     Op = DAG.getNode(ISD::ADD, dl, VT,
    2745          68 :                      DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
    2746             :                      DAG.getNode(ISD::AND, dl, VT,
    2747             :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2748             :                                              DAG.getConstant(2, dl, ShVT)),
    2749         204 :                                  Mask33));
    2750             :     // v = (v + (v >> 4)) & 0x0F0F0F0F...
    2751         136 :     Op = DAG.getNode(ISD::AND, dl, VT,
    2752             :                      DAG.getNode(ISD::ADD, dl, VT, Op,
    2753             :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2754             :                                              DAG.getConstant(4, dl, ShVT))),
    2755         136 :                      Mask0F);
    2756             :     // v = (v * 0x01010101...) >> (Len - 8)
    2757         136 :     Op = DAG.getNode(ISD::SRL, dl, VT,
    2758          68 :                      DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
    2759         204 :                      DAG.getConstant(Len - 8, dl, ShVT));
    2760             : 
    2761          68 :     return Op;
    2762             :   }
    2763         121 :   case ISD::CTLZ_ZERO_UNDEF:
    2764             :     // This trivially expands to CTLZ.
    2765         242 :     return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
    2766          62 :   case ISD::CTLZ: {
    2767          62 :     EVT VT = Op.getValueType();
    2768          62 :     unsigned Len = VT.getSizeInBits();
    2769             : 
    2770          62 :     if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
    2771          40 :       EVT SetCCVT = getSetCCResultType(VT);
    2772          80 :       SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
    2773          40 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    2774          40 :       SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
    2775          40 :       return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
    2776          40 :                          DAG.getConstant(Len, dl, VT), CTLZ);
    2777             :     }
    2778             : 
    2779             :     // for now, we do this:
    2780             :     // x = x | (x >> 1);
    2781             :     // x = x | (x >> 2);
    2782             :     // ...
    2783             :     // x = x | (x >>16);
    2784             :     // x = x | (x >>32); // for 64-bit input
    2785             :     // return popcount(~x);
    2786             :     //
    2787             :     // Ref: "Hacker's Delight" by Henry Warren
    2788          44 :     EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2789         250 :     for (unsigned i = 0; (1U << i) <= (Len / 2); ++i) {
    2790         114 :       SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
    2791         228 :       Op = DAG.getNode(ISD::OR, dl, VT, Op,
    2792         228 :                        DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
    2793             :     }
    2794          22 :     Op = DAG.getNOT(dl, Op, VT);
    2795          44 :     return DAG.getNode(ISD::CTPOP, dl, VT, Op);
    2796             :   }
    2797          87 :   case ISD::CTTZ_ZERO_UNDEF:
    2798             :     // This trivially expands to CTTZ.
    2799         174 :     return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
    2800          31 :   case ISD::CTTZ: {
    2801          31 :     EVT VT = Op.getValueType();
    2802          31 :     unsigned Len = VT.getSizeInBits();
    2803             : 
    2804          31 :     if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
    2805          17 :       EVT SetCCVT = getSetCCResultType(VT);
    2806          34 :       SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
    2807          17 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    2808          17 :       SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
    2809          17 :       return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
    2810          17 :                          DAG.getConstant(Len, dl, VT), CTTZ);
    2811             :     }
    2812             : 
    2813             :     // for now, we use: { return popcount(~x & (x - 1)); }
    2814             :     // unless the target has ctlz but not ctpop, in which case we use:
    2815             :     // { return 32 - nlz(~x & (x-1)); }
    2816             :     // Ref: "Hacker's Delight" by Henry Warren
    2817          14 :     SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
    2818          14 :                                DAG.getNOT(dl, Op, VT),
    2819             :                                DAG.getNode(ISD::SUB, dl, VT, Op,
    2820          28 :                                            DAG.getConstant(1, dl, VT)));
    2821             :     // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
    2822          14 :     if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
    2823             :         TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
    2824           5 :       return DAG.getNode(ISD::SUB, dl, VT,
    2825           5 :                          DAG.getConstant(VT.getSizeInBits(), dl, VT),
    2826          10 :                          DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
    2827          18 :     return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
    2828             :   }
    2829             :   }
    2830             : }
    2831             : 
    2832      239508 : bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
    2833             :   LLVM_DEBUG(dbgs() << "Trying to expand node\n");
    2834             :   SmallVector<SDValue, 8> Results;
    2835             :   SDLoc dl(Node);
    2836      239508 :   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
    2837             :   bool NeedInvert;
    2838      479016 :   switch (Node->getOpcode()) {
    2839         369 :   case ISD::CTPOP:
    2840             :   case ISD::CTLZ:
    2841             :   case ISD::CTLZ_ZERO_UNDEF:
    2842             :   case ISD::CTTZ:
    2843             :   case ISD::CTTZ_ZERO_UNDEF:
    2844         738 :     Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
    2845         369 :     Results.push_back(Tmp1);
    2846         369 :     break;
    2847          92 :   case ISD::BITREVERSE:
    2848         184 :     Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
    2849          92 :     break;
    2850         133 :   case ISD::BSWAP:
    2851         266 :     Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
    2852         133 :     break;
    2853           1 :   case ISD::FRAMEADDR:
    2854             :   case ISD::RETURNADDR:
    2855             :   case ISD::FRAME_TO_ARGS_OFFSET:
    2856           2 :     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
    2857           1 :     break;
    2858           6 :   case ISD::EH_DWARF_CFA: {
    2859          12 :     SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
    2860          12 :                                         TLI.getPointerTy(DAG.getDataLayout()));
    2861           6 :     SDValue Offset = DAG.getNode(ISD::ADD, dl,
    2862             :                                  CfaArg.getValueType(),
    2863             :                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
    2864             :                                              CfaArg.getValueType()),
    2865          12 :                                  CfaArg);
    2866           6 :     SDValue FA = DAG.getNode(
    2867           6 :         ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
    2868          24 :         DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
    2869          12 :     Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
    2870          12 :                                   FA, Offset));
    2871             :     break;
    2872             :   }
    2873           1 :   case ISD::FLT_ROUNDS_:
    2874           2 :     Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
    2875           1 :     break;
    2876         292 :   case ISD::EH_RETURN:
    2877             :   case ISD::EH_LABEL:
    2878             :   case ISD::PREFETCH:
    2879             :   case ISD::VAEND:
    2880             :   case ISD::EH_SJLJ_LONGJMP:
    2881             :     // If the target didn't expand these, there's nothing to do, so just
    2882             :     // preserve the chain and be done.
    2883         584 :     Results.push_back(Node->getOperand(0));
    2884         292 :     break;
    2885           3 :   case ISD::READCYCLECOUNTER:
    2886             :     // If the target didn't expand this, just return 'zero' and preserve the
    2887             :     // chain.
    2888           6 :     Results.append(Node->getNumValues() - 1,
    2889           9 :                    DAG.getConstant(0, dl, Node->getValueType(0)));
    2890           6 :     Results.push_back(Node->getOperand(0));
    2891           3 :     break;
    2892           0 :   case ISD::EH_SJLJ_SETJMP:
    2893             :     // If the target didn't expand this, just return 'zero' and preserve the
    2894             :     // chain.
    2895           0 :     Results.push_back(DAG.getConstant(0, dl, MVT::i32));
    2896           0 :     Results.push_back(Node->getOperand(0));
    2897           0 :     break;
    2898           6 :   case ISD::ATOMIC_LOAD: {
    2899             :     // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
    2900          12 :     SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
    2901          18 :     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
    2902           6 :     SDValue Swap = DAG.getAtomicCmpSwap(
    2903             :         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
    2904           6 :         Node->getOperand(0), Node->getOperand(1), Zero, Zero,
    2905          12 :         cast<AtomicSDNode>(Node)->getMemOperand());
    2906           6 :     Results.push_back(Swap.getValue(0));
    2907           6 :     Results.push_back(Swap.getValue(1));
    2908             :     break;
    2909             :   }
    2910          14 :   case ISD::ATOMIC_STORE: {
    2911             :     // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
    2912          14 :     SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
    2913             :                                  cast<AtomicSDNode>(Node)->getMemoryVT(),
    2914             :                                  Node->getOperand(0),
    2915          14 :                                  Node->getOperand(1), Node->getOperand(2),
    2916          28 :                                  cast<AtomicSDNode>(Node)->getMemOperand());
    2917          14 :     Results.push_back(Swap.getValue(1));
    2918             :     break;
    2919             :   }
    2920         941 :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
    2921             :     // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
    2922             :     // splits out the success value as a comparison. Expanding the resulting
    2923             :     // ATOMIC_CMP_SWAP will produce a libcall.
    2924        2823 :     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
    2925         941 :     SDValue Res = DAG.getAtomicCmpSwap(
    2926             :         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
    2927             :         Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
    2928        1882 :         Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
    2929             : 
    2930         941 :     SDValue ExtRes = Res;
    2931         941 :     SDValue LHS = Res;
    2932         941 :     SDValue RHS = Node->getOperand(1);
    2933             : 
    2934         941 :     EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
    2935        1882 :     EVT OuterType = Node->getValueType(0);
    2936         941 :     switch (TLI.getExtendForAtomicOps()) {
    2937          55 :     case ISD::SIGN_EXTEND:
    2938         110 :       LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
    2939         110 :                         DAG.getValueType(AtomicType));
    2940         110 :       RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
    2941         110 :                         Node->getOperand(2), DAG.getValueType(AtomicType));
    2942          55 :       ExtRes = LHS;
    2943          55 :       break;
    2944         835 :     case ISD::ZERO_EXTEND:
    2945        1670 :       LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
    2946        1670 :                         DAG.getValueType(AtomicType));
    2947        1670 :       RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
    2948         835 :       ExtRes = LHS;
    2949         835 :       break;
    2950          51 :     case ISD::ANY_EXTEND:
    2951          51 :       LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
    2952         102 :       RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
    2953          51 :       break;
    2954           0 :     default:
    2955           0 :       llvm_unreachable("Invalid atomic op extension");
    2956             :     }
    2957             : 
    2958             :     SDValue Success =
    2959        1882 :         DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
    2960             : 
    2961         941 :     Results.push_back(ExtRes.getValue(0));
    2962         941 :     Results.push_back(Success);
    2963         941 :     Results.push_back(Res.getValue(1));
    2964             :     break;
    2965             :   }
    2966         155 :   case ISD::DYNAMIC_STACKALLOC:
    2967         155 :     ExpandDYNAMIC_STACKALLOC(Node, Results);
    2968         155 :     break;
    2969             :   case ISD::MERGE_VALUES:
    2970     1243375 :     for (unsigned i = 0; i < Node->getNumValues(); i++)
    2971      466252 :       Results.push_back(Node->getOperand(i));
    2972             :     break;
    2973         442 :   case ISD::UNDEF: {
    2974         884 :     EVT VT = Node->getValueType(0);
    2975         442 :     if (VT.isInteger())
    2976         331 :       Results.push_back(DAG.getConstant(0, dl, VT));
    2977             :     else {
    2978             :       assert(VT.isFloatingPoint() && "Unknown value type!");
    2979         111 :       Results.push_back(DAG.getConstantFP(0, dl, VT));
    2980             :     }
    2981             :     break;
    2982             :   }
    2983         400 :   case ISD::FP_ROUND:
    2984             :   case ISD::BITCAST:
    2985         800 :     Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
    2986         400 :                             Node->getValueType(0), dl);
    2987         400 :     Results.push_back(Tmp1);
    2988         400 :     break;
    2989           0 :   case ISD::FP_EXTEND:
    2990           0 :     Tmp1 = EmitStackConvert(Node->getOperand(0),
    2991           0 :                             Node->getOperand(0).getValueType(),
    2992           0 :                             Node->getValueType(0), dl);
    2993           0 :     Results.push_back(Tmp1);
    2994           0 :     break;
    2995         813 :   case ISD::SIGN_EXTEND_INREG: {
    2996        1626 :     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
    2997        1626 :     EVT VT = Node->getValueType(0);
    2998             : 
    2999             :     // An in-register sign-extend of a boolean is a negation:
    3000             :     // 'true' (1) sign-extended is -1.
    3001             :     // 'false' (0) sign-extended is 0.
    3002             :     // However, we must mask the high bits of the source operand because the
    3003             :     // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
    3004             : 
    3005             :     // TODO: Do this for vectors too?
    3006         813 :     if (ExtraVT.getSizeInBits() == 1) {
    3007         245 :       SDValue One = DAG.getConstant(1, dl, VT);
    3008         735 :       SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
    3009         245 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    3010         490 :       SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
    3011         245 :       Results.push_back(Neg);
    3012             :       break;
    3013             :     }
    3014             : 
    3015             :     // NOTE: we could fall back on load/store here too for targets without
    3016             :     // SRA.  However, it is doubtful that any exist.
    3017        1136 :     EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    3018             :     unsigned BitsDiff = VT.getScalarSizeInBits() -
    3019         568 :                         ExtraVT.getScalarSizeInBits();
    3020         568 :     SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
    3021        1136 :     Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
    3022        1704 :                        Node->getOperand(0), ShiftCst);
    3023        1704 :     Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
    3024         568 :     Results.push_back(Tmp1);
    3025         568 :     break;
    3026             :   }
    3027           0 :   case ISD::FP_ROUND_INREG: {
    3028             :     // The only way we can lower this is to turn it into a TRUNCSTORE,
    3029             :     // EXTLOAD pair, targeting a temporary location (a stack slot).
    3030             : 
    3031             :     // NOTE: there is a choice here between constantly creating new stack
    3032             :     // slots and always reusing the same one.  We currently always create
    3033             :     // new ones, as reuse may inhibit scheduling.
    3034           0 :     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
    3035           0 :     Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
    3036           0 :                             Node->getValueType(0), dl);
    3037           0 :     Results.push_back(Tmp1);
    3038             :     break;
    3039             :   }
    3040         136 :   case ISD::SINT_TO_FP:
    3041             :   case ISD::UINT_TO_FP:
    3042         136 :     Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
    3043         408 :                                 Node->getOperand(0), Node->getValueType(0), dl);
    3044         136 :     Results.push_back(Tmp1);
    3045         136 :     break;
    3046          60 :   case ISD::FP_TO_SINT:
    3047          60 :     if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
    3048          60 :       Results.push_back(Tmp1);
    3049             :     break;
    3050         201 :   case ISD::FP_TO_UINT: {
    3051         201 :     SDValue True, False;
    3052         402 :     EVT VT =  Node->getOperand(0).getValueType();
    3053         402 :     EVT NVT = Node->getValueType(0);
    3054             :     APFloat apf(DAG.EVTToAPFloatSemantics(VT),
    3055         402 :                 APInt::getNullValue(VT.getSizeInBits()));
    3056         201 :     APInt x = APInt::getSignMask(NVT.getSizeInBits());
    3057         201 :     (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
    3058         201 :     Tmp1 = DAG.getConstantFP(apf, dl, VT);
    3059         402 :     Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
    3060         201 :                         Node->getOperand(0),
    3061         201 :                         Tmp1, ISD::SETLT);
    3062         603 :     True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
    3063             :     // TODO: Should any fast-math-flags be set for the FSUB?
    3064         402 :     False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
    3065             :                         DAG.getNode(ISD::FSUB, dl, VT,
    3066         603 :                                     Node->getOperand(0), Tmp1));
    3067         402 :     False = DAG.getNode(ISD::XOR, dl, NVT, False,
    3068         402 :                         DAG.getConstant(x, dl, NVT));
    3069         201 :     Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
    3070         201 :     Results.push_back(Tmp1);
    3071             :     break;
    3072             :   }
    3073          29 :   case ISD::VAARG:
    3074          29 :     Results.push_back(DAG.expandVAArg(Node));
    3075          29 :     Results.push_back(Results[0].getValue(1));
    3076          29 :     break;
    3077           3 :   case ISD::VACOPY:
    3078           3 :     Results.push_back(DAG.expandVACopy(Node));
    3079           3 :     break;
    3080        1856 :   case ISD::EXTRACT_VECTOR_ELT:
    3081        3712 :     if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
    3082             :       // This must be an access of the only element.  Return it.
    3083         132 :       Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
    3084         132 :                          Node->getOperand(0));
    3085             :     else
    3086        1790 :       Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
    3087        1856 :     Results.push_back(Tmp1);
    3088        1856 :     break;
    3089             :   case ISD::EXTRACT_SUBVECTOR:
    3090           0 :     Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
    3091           0 :     break;
    3092             :   case ISD::INSERT_SUBVECTOR:
    3093           0 :     Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
    3094           0 :     break;
    3095           0 :   case ISD::CONCAT_VECTORS:
    3096           0 :     Results.push_back(ExpandVectorBuildThroughStack(Node));
    3097           0 :     break;
    3098          18 :   case ISD::SCALAR_TO_VECTOR:
    3099          18 :     Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
    3100          18 :     break;
    3101        1060 :   case ISD::INSERT_VECTOR_ELT:
    3102        1060 :     Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
    3103             :                                               Node->getOperand(1),
    3104        1060 :                                               Node->getOperand(2), dl));
    3105        1060 :     break;
    3106             :   case ISD::VECTOR_SHUFFLE: {
    3107             :     SmallVector<int, 32> NewMask;
    3108             :     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
    3109             : 
    3110         300 :     EVT VT = Node->getValueType(0);
    3111         300 :     EVT EltVT = VT.getVectorElementType();
    3112         300 :     SDValue Op0 = Node->getOperand(0);
    3113         300 :     SDValue Op1 = Node->getOperand(1);
    3114         300 :     if (!TLI.isTypeLegal(EltVT)) {
    3115          14 :       EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
    3116             : 
    3117             :       // BUILD_VECTOR operands are allowed to be wider than the element type.
    3118             :       // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
    3119             :       // it.
    3120           7 :       if (NewEltVT.bitsLT(EltVT)) {
    3121             :         // Convert shuffle node.
    3122             :         // If original node was v4i64 and the new EltVT is i32,
    3123             :         // cast operands to v8i32 and re-build the mask.
    3124             : 
    3125             :         // Calculate new VT, the size of the new VT should be equal to original.
    3126             :         EVT NewVT =
    3127           0 :             EVT::getVectorVT(*DAG.getContext(), NewEltVT,
    3128           0 :                              VT.getSizeInBits() / NewEltVT.getSizeInBits());
    3129             :         assert(NewVT.bitsEq(VT));
    3130             : 
    3131             :         // cast operands to new VT
    3132           0 :         Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
    3133           0 :         Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
    3134             : 
    3135             :         // Convert the shuffle mask
    3136             :         unsigned int factor =
    3137           0 :                          NewVT.getVectorNumElements()/VT.getVectorNumElements();
    3138             : 
    3139             :         // EltVT gets smaller
    3140             :         assert(factor > 0);
    3141             : 
    3142           0 :         for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
    3143           0 :           if (Mask[i] < 0) {
    3144           0 :             for (unsigned fi = 0; fi < factor; ++fi)
    3145           0 :               NewMask.push_back(Mask[i]);
    3146             :           }
    3147             :           else {
    3148           0 :             for (unsigned fi = 0; fi < factor; ++fi)
    3149           0 :               NewMask.push_back(Mask[i]*factor+fi);
    3150             :           }
    3151             :         }
    3152             :         Mask = NewMask;
    3153           0 :         VT = NewVT;
    3154             :       }
    3155           7 :       EltVT = NewEltVT;
    3156             :     }
    3157         300 :     unsigned NumElems = VT.getVectorNumElements();
    3158             :     SmallVector<SDValue, 16> Ops;
    3159        2084 :     for (unsigned i = 0; i != NumElems; ++i) {
    3160        1837 :       if (Mask[i] < 0) {
    3161          53 :         Ops.push_back(DAG.getUNDEF(EltVT));
    3162          53 :         continue;
    3163             :       }
    3164         839 :       unsigned Idx = Mask[i];
    3165         839 :       if (Idx < NumElems)
    3166         976 :         Ops.push_back(DAG.getNode(
    3167             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
    3168        1952 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3169             :       else
    3170         702 :         Ops.push_back(DAG.getNode(
    3171             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
    3172         351 :             DAG.getConstant(Idx - NumElems, dl,
    3173        1404 :                             TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3174             :     }
    3175             : 
    3176         600 :     Tmp1 = DAG.getBuildVector(VT, dl, Ops);
    3177             :     // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
    3178         900 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
    3179         300 :     Results.push_back(Tmp1);
    3180             :     break;
    3181             :   }
    3182         391 :   case ISD::EXTRACT_ELEMENT: {
    3183         782 :     EVT OpTy = Node->getOperand(0).getValueType();
    3184         782 :     if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
    3185             :       // 1 -> Hi
    3186         531 :       Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
    3187         177 :                          DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
    3188         177 :                                          TLI.getShiftAmountTy(
    3189         177 :                                              Node->getOperand(0).getValueType(),
    3190         885 :                                              DAG.getDataLayout())));
    3191         531 :       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
    3192             :     } else {
    3193             :       // 0 -> Lo
    3194         428 :       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
    3195         428 :                          Node->getOperand(0));
    3196             :     }
    3197         391 :     Results.push_back(Tmp1);
    3198             :     break;
    3199             :   }
    3200         120 :   case ISD::STACKSAVE:
    3201             :     // Expand to CopyFromReg if the target set
    3202             :     // StackPointerRegisterToSaveRestore.
    3203         120 :     if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
    3204         360 :       Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
    3205         120 :                                            Node->getValueType(0)));
    3206         120 :       Results.push_back(Results[0].getValue(1));
    3207             :     } else {
    3208           0 :       Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
    3209           0 :       Results.push_back(Node->getOperand(0));
    3210             :     }
    3211             :     break;
    3212          27 :   case ISD::STACKRESTORE:
    3213             :     // Expand to CopyToReg if the target set
    3214             :     // StackPointerRegisterToSaveRestore.
    3215          27 :     if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
    3216          54 :       Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
    3217          27 :                                          Node->getOperand(1)));
    3218             :     } else {
    3219           0 :       Results.push_back(Node->getOperand(0));
    3220             :     }
    3221             :     break;
    3222           0 :   case ISD::GET_DYNAMIC_AREA_OFFSET:
    3223           0 :     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
    3224           0 :     Results.push_back(Results[0].getValue(0));
    3225           0 :     break;
    3226          42 :   case ISD::FCOPYSIGN:
    3227          42 :     Results.push_back(ExpandFCOPYSIGN(Node));
    3228          42 :     break;
    3229           1 :   case ISD::FNEG:
    3230             :     // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
    3231           2 :     Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
    3232             :     // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
    3233           2 :     Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
    3234           3 :                        Node->getOperand(0));
    3235           1 :     Results.push_back(Tmp1);
    3236           1 :     break;
    3237           2 :   case ISD::FABS:
    3238           2 :     Results.push_back(ExpandFABS(Node));
    3239           2 :     break;
    3240           0 :   case ISD::SMIN:
    3241             :   case ISD::SMAX:
    3242             :   case ISD::UMIN:
    3243             :   case ISD::UMAX: {
    3244             :     // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
    3245             :     ISD::CondCode Pred;
    3246           0 :     switch (Node->getOpcode()) {
    3247           0 :     default: llvm_unreachable("How did we get here?");
    3248             :     case ISD::SMAX: Pred = ISD::SETGT; break;
    3249           0 :     case ISD::SMIN: Pred = ISD::SETLT; break;
    3250           0 :     case ISD::UMAX: Pred = ISD::SETUGT; break;
    3251           0 :     case ISD::UMIN: Pred = ISD::SETULT; break;
    3252             :     }
    3253           0 :     Tmp1 = Node->getOperand(0);
    3254           0 :     Tmp2 = Node->getOperand(1);
    3255           0 :     Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
    3256           0 :     Results.push_back(Tmp1);
    3257           0 :     break;
    3258             :   }
    3259             : 
    3260         289 :   case ISD::FSIN:
    3261             :   case ISD::FCOS: {
    3262         578 :     EVT VT = Node->getValueType(0);
    3263             :     // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
    3264             :     // fcos which share the same operand and both are used.
    3265         558 :     if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
    3266             :          isSinCosLibcallAvailable(Node, TLI))
    3267         140 :         && useSinCos(Node)) {
    3268          78 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3269         156 :       Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
    3270          78 :       if (Node->getOpcode() == ISD::FCOS)
    3271          39 :         Tmp1 = Tmp1.getValue(1);
    3272          78 :       Results.push_back(Tmp1);
    3273             :     }
    3274             :     break;
    3275             :   }
    3276           0 :   case ISD::FMAD:
    3277           0 :     llvm_unreachable("Illegal fmad should never be formed");
    3278             : 
    3279             :   case ISD::FP16_TO_FP:
    3280         357 :     if (Node->getValueType(0) != MVT::f32) {
    3281             :       // We can extend to types bigger than f32 in two steps without changing
    3282             :       // the result. Since "f16 -> f32" is much more commonly available, give
    3283             :       // CodeGen the option of emitting that before resorting to a libcall.
    3284             :       SDValue Res =
    3285           0 :           DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
    3286           0 :       Results.push_back(
    3287           0 :           DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
    3288             :     }
    3289             :     break;
    3290         479 :   case ISD::FP_TO_FP16:
    3291             :     LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
    3292         479 :     if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
    3293          15 :       SDValue Op = Node->getOperand(0);
    3294             :       MVT SVT = Op.getSimpleValueType();
    3295          15 :       if ((SVT == MVT::f64 || SVT == MVT::f80) &&
    3296          15 :           TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
    3297             :         // Under fastmath, we can expand this node into a fround followed by
    3298             :         // a float-half conversion.
    3299          13 :         SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
    3300          26 :                                        DAG.getIntPtrConstant(0, dl));
    3301          13 :         Results.push_back(
    3302          52 :             DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
    3303             :       }
    3304             :     }
    3305             :     break;
    3306             :   case ISD::ConstantFP: {
    3307             :     ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
    3308             :     // Check to see if this FP immediate is already legal.
    3309             :     // If this is a legal constant, turn it into a TargetConstantFP node.
    3310       19617 :     if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
    3311        3606 :       Results.push_back(ExpandConstantFP(CFP, true));
    3312             :     break;
    3313             :   }
    3314             :   case ISD::Constant: {
    3315             :     ConstantSDNode *CP = cast<ConstantSDNode>(Node);
    3316           0 :     Results.push_back(ExpandConstant(CP));
    3317           0 :     break;
    3318             :   }
    3319         221 :   case ISD::FSUB: {
    3320         442 :     EVT VT = Node->getValueType(0);
    3321         221 :     if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
    3322             :         TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
    3323         172 :       const SDNodeFlags Flags = Node->getFlags();
    3324         516 :       Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
    3325         344 :       Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
    3326         172 :       Results.push_back(Tmp1);
    3327             :     }
    3328             :     break;
    3329             :   }
    3330           0 :   case ISD::SUB: {
    3331           0 :     EVT VT = Node->getValueType(0);
    3332             :     assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
    3333             :            TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
    3334             :            "Don't know how to expand this subtraction!");
    3335           0 :     Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
    3336           0 :                DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
    3337           0 :                                VT));
    3338           0 :     Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
    3339           0 :     Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
    3340             :     break;
    3341             :   }
    3342        1010 :   case ISD::UREM:
    3343             :   case ISD::SREM: {
    3344        2020 :     EVT VT = Node->getValueType(0);
    3345             :     bool isSigned = Node->getOpcode() == ISD::SREM;
    3346        1010 :     unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
    3347        1010 :     unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3348        1010 :     Tmp2 = Node->getOperand(0);
    3349        1010 :     Tmp3 = Node->getOperand(1);
    3350        1010 :     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
    3351         824 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3352         824 :       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
    3353         824 :       Results.push_back(Tmp1);
    3354         186 :     } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
    3355             :       // X % Y -> X-X/Y*Y
    3356         312 :       Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
    3357         312 :       Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
    3358         312 :       Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
    3359         156 :       Results.push_back(Tmp1);
    3360             :     }
    3361             :     break;
    3362             :   }
    3363         813 :   case ISD::UDIV:
    3364             :   case ISD::SDIV: {
    3365             :     bool isSigned = Node->getOpcode() == ISD::SDIV;
    3366         813 :     unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3367        1626 :     EVT VT = Node->getValueType(0);
    3368         813 :     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
    3369         804 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3370        1608 :       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
    3371         804 :                          Node->getOperand(1));
    3372         804 :       Results.push_back(Tmp1);
    3373             :     }
    3374             :     break;
    3375             :   }
    3376         412 :   case ISD::MULHU:
    3377             :   case ISD::MULHS: {
    3378             :     unsigned ExpandOpcode =
    3379         412 :         Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
    3380         824 :     EVT VT = Node->getValueType(0);
    3381         412 :     SDVTList VTs = DAG.getVTList(VT, VT);
    3382             : 
    3383         824 :     Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
    3384         412 :                        Node->getOperand(1));
    3385         412 :     Results.push_back(Tmp1.getValue(1));
    3386             :     break;
    3387             :   }
    3388         570 :   case ISD::UMUL_LOHI:
    3389             :   case ISD::SMUL_LOHI: {
    3390         570 :     SDValue LHS = Node->getOperand(0);
    3391         570 :     SDValue RHS = Node->getOperand(1);
    3392             :     MVT VT = LHS.getSimpleValueType();
    3393             :     unsigned MULHOpcode =
    3394         570 :         Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
    3395             : 
    3396         570 :     if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
    3397         912 :       Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
    3398         912 :       Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
    3399         456 :       break;
    3400             :     }
    3401             : 
    3402             :     SmallVector<SDValue, 4> Halves;
    3403         114 :     EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
    3404             :     assert(TLI.isTypeLegal(HalfType));
    3405         570 :     if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
    3406             :                            HalfType, DAG,
    3407         114 :                            TargetLowering::MulExpansionKind::Always)) {
    3408         570 :       for (unsigned i = 0; i < 2; ++i) {
    3409         684 :         SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
    3410         684 :         SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
    3411         228 :         SDValue Shift = DAG.getConstant(
    3412             :             HalfType.getScalarSizeInBits(), dl,
    3413         684 :             TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
    3414         456 :         Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
    3415         456 :         Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
    3416             :       }
    3417             :       break;
    3418             :     }
    3419             :     break;
    3420             :   }
    3421         185 :   case ISD::MUL: {
    3422         370 :     EVT VT = Node->getValueType(0);
    3423         185 :     SDVTList VTs = DAG.getVTList(VT, VT);
    3424             :     // See if multiply or divide can be lowered using two-result operations.
    3425             :     // We just need the low half of the multiply; try both the signed
    3426             :     // and unsigned forms. If the target supports both SMUL_LOHI and
    3427             :     // UMUL_LOHI, form a preference by checking which forms of plain
    3428             :     // MULH it supports.
    3429         185 :     bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
    3430             :     bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
    3431             :     bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
    3432             :     bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
    3433             :     unsigned OpToUse = 0;
    3434         185 :     if (HasSMUL_LOHI && !HasMULHS) {
    3435             :       OpToUse = ISD::SMUL_LOHI;
    3436         172 :     } else if (HasUMUL_LOHI && !HasMULHU) {
    3437             :       OpToUse = ISD::UMUL_LOHI;
    3438         172 :     } else if (HasSMUL_LOHI) {
    3439             :       OpToUse = ISD::SMUL_LOHI;
    3440         172 :     } else if (HasUMUL_LOHI) {
    3441             :       OpToUse = ISD::UMUL_LOHI;
    3442             :     }
    3443             :     if (OpToUse) {
    3444          26 :       Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
    3445          13 :                                     Node->getOperand(1)));
    3446          13 :       break;
    3447             :     }
    3448             : 
    3449         172 :     SDValue Lo, Hi;
    3450         172 :     EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
    3451         172 :     if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
    3452             :         TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
    3453             :         TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
    3454         172 :         TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
    3455         178 :         TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
    3456         172 :                       TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
    3457         332 :       Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
    3458         332 :       Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
    3459             :       SDValue Shift =
    3460         332 :           DAG.getConstant(HalfType.getSizeInBits(), dl,
    3461         498 :                           TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
    3462         332 :       Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
    3463         332 :       Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
    3464             :     }
    3465             :     break;
    3466             :   }
    3467          35 :   case ISD::SADDO:
    3468             :   case ISD::SSUBO: {
    3469          35 :     SDValue LHS = Node->getOperand(0);
    3470          35 :     SDValue RHS = Node->getOperand(1);
    3471          35 :     SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
    3472             :                               ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
    3473          35 :                               LHS, RHS);
    3474          35 :     Results.push_back(Sum);
    3475          70 :     EVT ResultType = Node->getValueType(1);
    3476          35 :     EVT OType = getSetCCResultType(Node->getValueType(0));
    3477             : 
    3478          70 :     SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
    3479             : 
    3480             :     //   LHSSign -> LHS >= 0
    3481             :     //   RHSSign -> RHS >= 0
    3482             :     //   SumSign -> Sum >= 0
    3483             :     //
    3484             :     //   Add:
    3485             :     //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
    3486             :     //   Sub:
    3487             :     //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
    3488          35 :     SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
    3489          35 :     SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
    3490          35 :     SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
    3491          35 :                                       Node->getOpcode() == ISD::SADDO ?
    3492          35 :                                       ISD::SETEQ : ISD::SETNE);
    3493             : 
    3494          35 :     SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
    3495          35 :     SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
    3496             : 
    3497          70 :     SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
    3498          35 :     Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
    3499             :     break;
    3500             :   }
    3501          25 :   case ISD::UADDO:
    3502             :   case ISD::USUBO: {
    3503          25 :     SDValue LHS = Node->getOperand(0);
    3504          25 :     SDValue RHS = Node->getOperand(1);
    3505             :     bool IsAdd = Node->getOpcode() == ISD::UADDO;
    3506             :     // If ADD/SUBCARRY is legal, use that instead.
    3507          25 :     unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
    3508          50 :     if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
    3509           6 :       SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
    3510           3 :       SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
    3511           9 :                                       { LHS, RHS, CarryIn });
    3512           3 :       Results.push_back(SDValue(NodeCarry.getNode(), 0));
    3513           3 :       Results.push_back(SDValue(NodeCarry.getNode(), 1));
    3514             :       break;
    3515             :     }
    3516             : 
    3517          22 :     SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
    3518          22 :                               LHS.getValueType(), LHS, RHS);
    3519          22 :     Results.push_back(Sum);
    3520             : 
    3521          44 :     EVT ResultType = Node->getValueType(1);
    3522          22 :     EVT SetCCType = getSetCCResultType(Node->getValueType(0));
    3523          22 :     ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
    3524          22 :     SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
    3525             : 
    3526          22 :     Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
    3527          22 :     break;
    3528             :   }
    3529          17 :   case ISD::UMULO:
    3530             :   case ISD::SMULO: {
    3531          34 :     EVT VT = Node->getValueType(0);
    3532          17 :     EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
    3533          17 :     SDValue LHS = Node->getOperand(0);
    3534          17 :     SDValue RHS = Node->getOperand(1);
    3535          17 :     SDValue BottomHalf;
    3536          17 :     SDValue TopHalf;
    3537             :     static const unsigned Ops[2][3] =
    3538             :         { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
    3539             :           { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
    3540          34 :     bool isSigned = Node->getOpcode() == ISD::SMULO;
    3541          17 :     if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
    3542          10 :       BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
    3543          10 :       TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
    3544          12 :     } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
    3545           8 :       BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
    3546           4 :                                RHS);
    3547           4 :       TopHalf = BottomHalf.getValue(1);
    3548           8 :     } else if (TLI.isTypeLegal(WideVT)) {
    3549           0 :       LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
    3550           0 :       RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
    3551           0 :       Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
    3552           0 :       BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
    3553           0 :                                DAG.getIntPtrConstant(0, dl));
    3554           0 :       TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
    3555           0 :                             DAG.getIntPtrConstant(1, dl));
    3556             :     } else {
    3557             :       // We can fall back to a libcall with an illegal type for the MUL if we
    3558             :       // have a libcall big enough.
    3559             :       // Also, we can fall back to a division in some cases, but that's a big
    3560             :       // performance hit in the general case.
    3561             :       RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    3562             :       if (WideVT == MVT::i16)
    3563             :         LC = RTLIB::MUL_I16;
    3564             :       else if (WideVT == MVT::i32)
    3565             :         LC = RTLIB::MUL_I32;
    3566             :       else if (WideVT == MVT::i64)
    3567             :         LC = RTLIB::MUL_I64;
    3568             :       else if (WideVT == MVT::i128)
    3569             :         LC = RTLIB::MUL_I128;
    3570             :       assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
    3571             : 
    3572             :       SDValue HiLHS;
    3573             :       SDValue HiRHS;
    3574           8 :       if (isSigned) {
    3575             :         // The high part is obtained by SRA'ing all but one of the bits of low
    3576             :         // part.
    3577           1 :         unsigned LoSize = VT.getSizeInBits();
    3578           1 :         HiLHS =
    3579           3 :             DAG.getNode(ISD::SRA, dl, VT, LHS,
    3580           1 :                         DAG.getConstant(LoSize - 1, dl,
    3581           3 :                                         TLI.getPointerTy(DAG.getDataLayout())));
    3582           1 :         HiRHS =
    3583           3 :             DAG.getNode(ISD::SRA, dl, VT, RHS,
    3584             :                         DAG.getConstant(LoSize - 1, dl,
    3585           3 :                                         TLI.getPointerTy(DAG.getDataLayout())));
    3586             :       } else {
    3587           7 :           HiLHS = DAG.getConstant(0, dl, VT);
    3588           7 :           HiRHS = DAG.getConstant(0, dl, VT);
    3589             :       }
    3590             : 
    3591             :       // Here we're passing the 2 arguments explicitly as 4 arguments that are
    3592             :       // pre-lowered to the correct types. This all depends upon WideVT not
    3593             :       // being a legal type for the architecture and thus has to be split to
    3594             :       // two arguments.
    3595             :       SDValue Ret;
    3596          16 :       if(DAG.getDataLayout().isLittleEndian()) {
    3597             :         // Halves of WideVT are packed into registers in different order
    3598             :         // depending on platform endianness. This is usually handled by
    3599             :         // the C calling convention, but we can't defer to it in
    3600             :         // the legalizer.
    3601           8 :         SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
    3602           8 :         Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
    3603             :       } else {
    3604           0 :         SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
    3605           0 :         Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
    3606             :       }
    3607             :       assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
    3608             :              "Ret value is a collection of constituent nodes holding result.");
    3609           8 :       BottomHalf = Ret.getOperand(0);
    3610           8 :       TopHalf = Ret.getOperand(1);
    3611             :     }
    3612             : 
    3613          17 :     if (isSigned) {
    3614          14 :       Tmp1 = DAG.getConstant(
    3615           7 :           VT.getSizeInBits() - 1, dl,
    3616          28 :           TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
    3617          14 :       Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
    3618          14 :       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
    3619           7 :                              ISD::SETNE);
    3620             :     } else {
    3621          20 :       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
    3622          10 :                              DAG.getConstant(0, dl, VT), ISD::SETNE);
    3623             :     }
    3624             : 
    3625             :     // Truncate the result if SetCC returns a larger type than needed.
    3626          34 :     EVT RType = Node->getValueType(1);
    3627          17 :     if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
    3628           2 :       TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
    3629             : 
    3630             :     assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
    3631             :            "Unexpected result type for S/UMULO legalization");
    3632             : 
    3633          17 :     Results.push_back(BottomHalf);
    3634          17 :     Results.push_back(TopHalf);
    3635             :     break;
    3636             :   }
    3637          23 :   case ISD::BUILD_PAIR: {
    3638          46 :     EVT PairTy = Node->getValueType(0);
    3639          69 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
    3640          69 :     Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
    3641          46 :     Tmp2 = DAG.getNode(
    3642             :         ISD::SHL, dl, PairTy, Tmp2,
    3643          23 :         DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
    3644          92 :                         TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
    3645          46 :     Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
    3646             :     break;
    3647             :   }
    3648         215 :   case ISD::SELECT:
    3649         215 :     Tmp1 = Node->getOperand(0);
    3650         215 :     Tmp2 = Node->getOperand(1);
    3651         215 :     Tmp3 = Node->getOperand(2);
    3652         430 :     if (Tmp1.getOpcode() == ISD::SETCC) {
    3653         312 :       Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
    3654             :                              Tmp2, Tmp3,
    3655         156 :                              cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
    3656             :     } else {
    3657         118 :       Tmp1 = DAG.getSelectCC(dl, Tmp1,
    3658             :                              DAG.getConstant(0, dl, Tmp1.getValueType()),
    3659         118 :                              Tmp2, Tmp3, ISD::SETNE);
    3660             :     }
    3661         215 :     Results.push_back(Tmp1);
    3662         215 :     break;
    3663         213 :   case ISD::BR_JT: {
    3664         213 :     SDValue Chain = Node->getOperand(0);
    3665         213 :     SDValue Table = Node->getOperand(1);
    3666         213 :     SDValue Index = Node->getOperand(2);
    3667             : 
    3668         213 :     const DataLayout &TD = DAG.getDataLayout();
    3669             :     EVT PTy = TLI.getPointerTy(TD);
    3670             : 
    3671             :     unsigned EntrySize =
    3672         213 :       DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
    3673             : 
    3674             :     // For power-of-two jumptable entry sizes convert multiplication to a shift.
    3675             :     // This transformation needs to be done here since otherwise the MIPS
    3676             :     // backend will end up emitting a three instruction multiply sequence
    3677             :     // instead of a single shift and MSP430 will call a runtime function.
    3678             :     if (llvm::isPowerOf2_32(EntrySize))
    3679         426 :       Index = DAG.getNode(
    3680             :           ISD::SHL, dl, Index.getValueType(), Index,
    3681         426 :           DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
    3682             :     else
    3683           0 :       Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
    3684           0 :                           DAG.getConstant(EntrySize, dl, Index.getValueType()));
    3685         213 :     SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
    3686         213 :                                Index, Table);
    3687             : 
    3688         213 :     EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
    3689         213 :     SDValue LD = DAG.getExtLoad(
    3690             :         ISD::SEXTLOAD, dl, PTy, Chain, Addr,
    3691         213 :         MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
    3692         213 :     Addr = LD;
    3693         213 :     if (TLI.isJumpTableRelative()) {
    3694             :       // For PIC, the sequence is:
    3695             :       // BRIND(load(Jumptable + index) + RelocBase)
    3696             :       // RelocBase can be JumpTable, GOT or some sort of global base.
    3697         120 :       Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
    3698         120 :                           TLI.getPICJumpTableRelocBase(Table, DAG));
    3699             :     }
    3700             : 
    3701         213 :     Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
    3702         213 :     Results.push_back(Tmp1);
    3703             :     break;
    3704             :   }
    3705         487 :   case ISD::BRCOND:
    3706             :     // Expand brcond's setcc into its constituent parts and create a BR_CC
    3707             :     // Node.
    3708         487 :     Tmp1 = Node->getOperand(0);
    3709         487 :     Tmp2 = Node->getOperand(1);
    3710         974 :     if (Tmp2.getOpcode() == ISD::SETCC) {
    3711           2 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
    3712             :                          Tmp1, Tmp2.getOperand(2),
    3713             :                          Tmp2.getOperand(0), Tmp2.getOperand(1),
    3714           1 :                          Node->getOperand(2));
    3715             :     } else {
    3716             :       // We test only the i1 bit.  Skip the AND if UNDEF or another AND.
    3717         486 :       if (Tmp2.isUndef() ||
    3718             :           (Tmp2.getOpcode() == ISD::AND &&
    3719          92 :            isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
    3720          92 :            cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
    3721          92 :         Tmp3 = Tmp2;
    3722             :       else
    3723         788 :         Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
    3724        1182 :                            DAG.getConstant(1, dl, Tmp2.getValueType()));
    3725         972 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
    3726         486 :                          DAG.getCondCode(ISD::SETNE), Tmp3,
    3727             :                          DAG.getConstant(0, dl, Tmp3.getValueType()),
    3728        2430 :                          Node->getOperand(2));
    3729             :     }
    3730         487 :     Results.push_back(Tmp1);
    3731         487 :     break;
    3732        2336 :   case ISD::SETCC: {
    3733        2336 :     Tmp1 = Node->getOperand(0);
    3734        2336 :     Tmp2 = Node->getOperand(1);
    3735        2336 :     Tmp3 = Node->getOperand(2);
    3736        2336 :     bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
    3737        2336 :                                            Tmp3, NeedInvert, dl);
    3738             : 
    3739        2336 :     if (Legalized) {
    3740             :       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
    3741             :       // condition code, create a new SETCC node.
    3742         328 :       if (Tmp3.getNode())
    3743         458 :         Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
    3744         458 :                            Tmp1, Tmp2, Tmp3);
    3745             : 
    3746             :       // If we expanded the SETCC by inverting the condition code, then wrap
    3747             :       // the existing SETCC in a NOT to restore the intended condition.
    3748         328 :       if (NeedInvert)
    3749         118 :         Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
    3750             : 
    3751         328 :       Results.push_back(Tmp1);
    3752         328 :       break;
    3753             :     }
    3754             : 
    3755             :     // Otherwise, SETCC for the given comparison type must be completely
    3756             :     // illegal; expand it into a SELECT_CC.
    3757        4016 :     EVT VT = Node->getValueType(0);
    3758             :     int TrueValue;
    3759        4016 :     switch (TLI.getBooleanContents(Tmp1.getValueType())) {
    3760         650 :     case TargetLowering::ZeroOrOneBooleanContent:
    3761             :     case TargetLowering::UndefinedBooleanContent:
    3762             :       TrueValue = 1;
    3763         650 :       break;
    3764        1358 :     case TargetLowering::ZeroOrNegativeOneBooleanContent:
    3765             :       TrueValue = -1;
    3766        1358 :       break;
    3767             :     }
    3768        4016 :     Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
    3769        2008 :                        DAG.getConstant(TrueValue, dl, VT),
    3770             :                        DAG.getConstant(0, dl, VT),
    3771        4016 :                        Tmp3);
    3772        2008 :     Results.push_back(Tmp1);
    3773        2008 :     break;
    3774             :   }
    3775        2215 :   case ISD::SELECT_CC: {
    3776        2215 :     Tmp1 = Node->getOperand(0);   // LHS
    3777        2215 :     Tmp2 = Node->getOperand(1);   // RHS
    3778        2215 :     Tmp3 = Node->getOperand(2);   // True
    3779        2215 :     Tmp4 = Node->getOperand(3);   // False
    3780        4430 :     EVT VT = Node->getValueType(0);
    3781        2215 :     SDValue CC = Node->getOperand(4);
    3782        2215 :     ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
    3783             : 
    3784        2215 :     if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
    3785             :       // If the condition code is legal, then we need to expand this
    3786             :       // node using SETCC and SELECT.
    3787        1914 :       EVT CmpVT = Tmp1.getValueType();
    3788             :       assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
    3789             :              "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
    3790             :              "expanded.");
    3791             :       EVT CCVT =
    3792        3828 :           TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
    3793        3828 :       SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
    3794        1914 :       Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
    3795             :       break;
    3796             :     }
    3797             : 
    3798             :     // SELECT_CC is legal, so the condition code must not be.
    3799             :     bool Legalized = false;
    3800             :     // Try to legalize by inverting the condition.  This is for targets that
    3801             :     // might support an ordered version of a condition, but not the unordered
    3802             :     // version (or vice versa).
    3803         301 :     ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
    3804         602 :                                                Tmp1.getValueType().isInteger());
    3805         301 :     if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
    3806             :       // Use the new condition code and swap true and false
    3807             :       Legalized = true;
    3808         256 :       Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
    3809             :     } else {
    3810             :       // If The inverse is not legal, then try to swap the arguments using
    3811             :       // the inverse condition code.
    3812          45 :       ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
    3813          45 :       if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
    3814             :         // The swapped inverse condition is legal, so swap true and false,
    3815             :         // lhs and rhs.
    3816             :         Legalized = true;
    3817          12 :         Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
    3818             :       }
    3819             :     }
    3820             : 
    3821             :     if (!Legalized) {
    3822          33 :       Legalized = LegalizeSetCCCondCode(
    3823             :           getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
    3824             :           dl);
    3825             : 
    3826             :       assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
    3827             : 
    3828             :       // If we expanded the SETCC by inverting the condition code, then swap
    3829             :       // the True/False operands to match.
    3830          33 :       if (NeedInvert)
    3831             :         std::swap(Tmp3, Tmp4);
    3832             : 
    3833             :       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
    3834             :       // condition code, create a new SELECT_CC node.
    3835          33 :       if (CC.getNode()) {
    3836          26 :         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
    3837          13 :                            Tmp1, Tmp2, Tmp3, Tmp4, CC);
    3838             :       } else {
    3839          40 :         Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
    3840          20 :         CC = DAG.getCondCode(ISD::SETNE);
    3841          40 :         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
    3842          20 :                            Tmp2, Tmp3, Tmp4, CC);
    3843             :       }
    3844             :     }
    3845         301 :     Results.push_back(Tmp1);
    3846         301 :     break;
    3847             :   }
    3848           9 :   case ISD::BR_CC: {
    3849           9 :     Tmp1 = Node->getOperand(0);              // Chain
    3850           9 :     Tmp2 = Node->getOperand(2);              // LHS
    3851           9 :     Tmp3 = Node->getOperand(3);              // RHS
    3852           9 :     Tmp4 = Node->getOperand(1);              // CC
    3853             : 
    3854          18 :     bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
    3855           9 :         Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
    3856             :     (void)Legalized;
    3857             :     assert(Legalized && "Can't legalize BR_CC with legal condition!");
    3858             : 
    3859             :     // If we expanded the SETCC by inverting the condition code, then wrap
    3860             :     // the existing SETCC in a NOT to restore the intended condition.
    3861           9 :     if (NeedInvert)
    3862           0 :       Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
    3863             : 
    3864             :     // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
    3865             :     // node.
    3866           9 :     if (Tmp4.getNode()) {
    3867           0 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
    3868           0 :                          Tmp4, Tmp2, Tmp3, Node->getOperand(4));
    3869             :     } else {
    3870          18 :       Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
    3871           9 :       Tmp4 = DAG.getCondCode(ISD::SETNE);
    3872          18 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
    3873          18 :                          Tmp2, Tmp3, Node->getOperand(4));
    3874             :     }
    3875           9 :     Results.push_back(Tmp1);
    3876           9 :     break;
    3877             :   }
    3878       21842 :   case ISD::BUILD_VECTOR:
    3879       21842 :     Results.push_back(ExpandBUILD_VECTOR(Node));
    3880       21842 :     break;
    3881          38 :   case ISD::SRA:
    3882             :   case ISD::SRL:
    3883             :   case ISD::SHL: {
    3884             :     // Scalarize vector SRA/SRL/SHL.
    3885          76 :     EVT VT = Node->getValueType(0);
    3886             :     assert(VT.isVector() && "Unable to legalize non-vector shift");
    3887             :     assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
    3888          38 :     unsigned NumElem = VT.getVectorNumElements();
    3889             : 
    3890             :     SmallVector<SDValue, 8> Scalars;
    3891         246 :     for (unsigned Idx = 0; Idx < NumElem; Idx++) {
    3892         104 :       SDValue Ex = DAG.getNode(
    3893         104 :           ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
    3894         416 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3895         104 :       SDValue Sh = DAG.getNode(
    3896         104 :           ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
    3897         416 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3898         208 :       Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
    3899         208 :                                     VT.getScalarType(), Ex, Sh));
    3900             :     }
    3901             : 
    3902         114 :     SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
    3903          38 :     ReplaceNode(SDValue(Node, 0), Result);
    3904             :     break;
    3905             :   }
    3906           4 :   case ISD::ROTL:
    3907             :   case ISD::ROTR: {
    3908             :     bool IsLeft = Node->getOpcode() == ISD::ROTL;
    3909           4 :     SDValue Op0 = Node->getOperand(0), Op1 = Node->getOperand(1);
    3910           8 :     EVT ResVT = Node->getValueType(0);
    3911           4 :     EVT OpVT = Op0.getValueType();
    3912             :     assert(OpVT == ResVT &&
    3913             :            "The result and the operand types of rotate should match");
    3914           4 :     EVT ShVT = Op1.getValueType();
    3915           8 :     SDValue Width = DAG.getConstant(OpVT.getScalarSizeInBits(), dl, ShVT);
    3916             : 
    3917             :     // If a rotate in the other direction is legal, use it.
    3918           4 :     unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
    3919           4 :     if (TLI.isOperationLegal(RevRot, ResVT)) {
    3920           0 :       SDValue Sub = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
    3921           0 :       Results.push_back(DAG.getNode(RevRot, dl, ResVT, Op0, Sub));
    3922             :       break;
    3923             :     }
    3924             : 
    3925             :     // Otherwise,
    3926             :     //   (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
    3927             :     //   (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
    3928             :     //
    3929             :     assert(isPowerOf2_32(OpVT.getScalarSizeInBits()) &&
    3930             :            "Expecting the type bitwidth to be a power of 2");
    3931           4 :     unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
    3932           4 :     unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
    3933           4 :     SDValue Width1 = DAG.getNode(ISD::SUB, dl, ShVT,
    3934           4 :                                  Width, DAG.getConstant(1, dl, ShVT));
    3935           8 :     SDValue NegOp1 = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
    3936           8 :     SDValue And0 = DAG.getNode(ISD::AND, dl, ShVT, Op1, Width1);
    3937           8 :     SDValue And1 = DAG.getNode(ISD::AND, dl, ShVT, NegOp1, Width1);
    3938             : 
    3939           4 :     SDValue Or = DAG.getNode(ISD::OR, dl, ResVT,
    3940           4 :                              DAG.getNode(ShOpc, dl, ResVT, Op0, And0),
    3941           8 :                              DAG.getNode(HsOpc, dl, ResVT, Op0, And1));
    3942           4 :     Results.push_back(Or);
    3943           4 :     break;
    3944             :   }
    3945             : 
    3946             :   case ISD::GLOBAL_OFFSET_TABLE:
    3947             :   case ISD::GlobalAddress:
    3948             :   case ISD::GlobalTLSAddress:
    3949             :   case ISD::ExternalSymbol:
    3950             :   case ISD::ConstantPool:
    3951             :   case ISD::JumpTable:
    3952             :   case ISD::INTRINSIC_W_CHAIN:
    3953             :   case ISD::INTRINSIC_WO_CHAIN:
    3954             :   case ISD::INTRINSIC_VOID:
    3955             :     // FIXME: Custom lowering for these operations shouldn't return null!
    3956             :     break;
    3957             :   }
    3958             : 
    3959             :   // Replace the original node with the legalized result.
    3960      239508 :   if (Results.empty()) {
    3961             :     LLVM_DEBUG(dbgs() << "Cannot expand node\n");
    3962             :     return false;
    3963             :   }
    3964             : 
    3965             :   LLVM_DEBUG(dbgs() << "Succesfully expanded node\n");
    3966      197530 :   ReplaceNode(Node, Results.data());
    3967      197530 :   return true;
    3968             : }
    3969             : 
    3970       42049 : void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
    3971             :   LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
    3972             :   SmallVector<SDValue, 8> Results;
    3973             :   SDLoc dl(Node);
    3974             :   // FIXME: Check flags on the node to see if we can use a finite call.
    3975       42049 :   bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
    3976       42049 :   unsigned Opc = Node->getOpcode();
    3977       42049 :   switch (Opc) {
    3978             :   case ISD::ATOMIC_FENCE: {
    3979             :     // If the target didn't lower this, lower it to '__sync_synchronize()' call
    3980             :     // FIXME: handle "fence singlethread" more efficiently.
    3981             :     TargetLowering::ArgListTy Args;
    3982             : 
    3983          26 :     TargetLowering::CallLoweringInfo CLI(DAG);
    3984             :     CLI.setDebugLoc(dl)
    3985          13 :         .setChain(Node->getOperand(0))
    3986          13 :         .setLibCallee(
    3987          13 :             CallingConv::C, Type::getVoidTy(*DAG.getContext()),
    3988          13 :             DAG.getExternalSymbol("__sync_synchronize",
    3989             :                                   TLI.getPointerTy(DAG.getDataLayout())),
    3990          52 :             std::move(Args));
    3991             : 
    3992          13 :     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
    3993             : 
    3994          13 :     Results.push_back(CallResult.second);
    3995             :     break;
    3996             :   }
    3997             :   // By default, atomic intrinsics are marked Legal and lowered. Targets
    3998             :   // which don't support them directly, however, may want libcalls, in which
    3999             :   // case they mark them Expand, and we get here.
    4000             :   case ISD::ATOMIC_SWAP:
    4001             :   case ISD::ATOMIC_LOAD_ADD:
    4002             :   case ISD::ATOMIC_LOAD_SUB:
    4003             :   case ISD::ATOMIC_LOAD_AND:
    4004             :   case ISD::ATOMIC_LOAD_CLR:
    4005             :   case ISD::ATOMIC_LOAD_OR:
    4006             :   case ISD::ATOMIC_LOAD_XOR:
    4007             :   case ISD::ATOMIC_LOAD_NAND:
    4008             :   case ISD::ATOMIC_LOAD_MIN:
    4009             :   case ISD::ATOMIC_LOAD_MAX:
    4010             :   case ISD::ATOMIC_LOAD_UMIN:
    4011             :   case ISD::ATOMIC_LOAD_UMAX:
    4012             :   case ISD::ATOMIC_CMP_SWAP: {
    4013         325 :     MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
    4014         325 :     RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
    4015             :     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
    4016             : 
    4017         325 :     std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
    4018         325 :     Results.push_back(Tmp.first);
    4019         325 :     Results.push_back(Tmp.second);
    4020             :     break;
    4021             :   }
    4022             :   case ISD::TRAP: {
    4023             :     // If this operation is not supported, lower it to 'abort()' call
    4024             :     TargetLowering::ArgListTy Args;
    4025           4 :     TargetLowering::CallLoweringInfo CLI(DAG);
    4026             :     CLI.setDebugLoc(dl)
    4027           2 :         .setChain(Node->getOperand(0))
    4028           4 :         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
    4029           2 :                       DAG.getExternalSymbol(
    4030             :                           "abort", TLI.getPointerTy(DAG.getDataLayout())),
    4031           8 :                       std::move(Args));
    4032           2 :     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
    4033             : 
    4034           2 :     Results.push_back(CallResult.second);
    4035             :     break;
    4036             :   }
    4037          45 :   case ISD::FMINNUM:
    4038          45 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
    4039             :                                       RTLIB::FMIN_F80, RTLIB::FMIN_F128,
    4040             :                                       RTLIB::FMIN_PPCF128));
    4041          45 :     break;
    4042          47 :   case ISD::FMAXNUM:
    4043          47 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
    4044             :                                       RTLIB::FMAX_F80, RTLIB::FMAX_F128,
    4045             :                                       RTLIB::FMAX_PPCF128));
    4046          47 :     break;
    4047           4 :   case ISD::FSQRT:
    4048             :   case ISD::STRICT_FSQRT:
    4049           4 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
    4050             :                                       RTLIB::SQRT_F80, RTLIB::SQRT_F128,
    4051             :                                       RTLIB::SQRT_PPCF128));
    4052           4 :     break;
    4053         122 :   case ISD::FSIN:
    4054             :   case ISD::STRICT_FSIN:
    4055         122 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
    4056             :                                       RTLIB::SIN_F80, RTLIB::SIN_F128,
    4057             :                                       RTLIB::SIN_PPCF128));
    4058         122 :     break;
    4059         101 :   case ISD::FCOS:
    4060             :   case ISD::STRICT_FCOS:
    4061         101 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
    4062             :                                       RTLIB::COS_F80, RTLIB::COS_F128,
    4063             :                                       RTLIB::COS_PPCF128));
    4064         101 :     break;
    4065          30 :   case ISD::FSINCOS:
    4066             :     // Expand into sincos libcall.
    4067          30 :     ExpandSinCosLibCall(Node, Results);
    4068          30 :     break;
    4069          74 :   case ISD::FLOG:
    4070             :   case ISD::STRICT_FLOG:
    4071          85 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
    4072           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
    4073             :                                         RTLIB::LOG_FINITE_F64,
    4074             :                                         RTLIB::LOG_FINITE_F80,
    4075             :                                         RTLIB::LOG_FINITE_F128,
    4076             :                                         RTLIB::LOG_FINITE_PPCF128));
    4077             :     else
    4078          70 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
    4079             :                                         RTLIB::LOG_F80, RTLIB::LOG_F128,
    4080             :                                         RTLIB::LOG_PPCF128));
    4081             :     break;
    4082          77 :   case ISD::FLOG2:
    4083             :   case ISD::STRICT_FLOG2:
    4084          88 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
    4085           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
    4086             :                                         RTLIB::LOG2_FINITE_F64,
    4087             :                                         RTLIB::LOG2_FINITE_F80,
    4088             :                                         RTLIB::LOG2_FINITE_F128,
    4089             :                                         RTLIB::LOG2_FINITE_PPCF128));
    4090             :     else
    4091          73 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
    4092             :                                         RTLIB::LOG2_F80, RTLIB::LOG2_F128,
    4093             :                                         RTLIB::LOG2_PPCF128));
    4094             :     break;
    4095          79 :   case ISD::FLOG10:
    4096             :   case ISD::STRICT_FLOG10:
    4097          90 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
    4098           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
    4099             :                                         RTLIB::LOG10_FINITE_F64,
    4100             :                                         RTLIB::LOG10_FINITE_F80,
    4101             :                                         RTLIB::LOG10_FINITE_F128,
    4102             :                                         RTLIB::LOG10_FINITE_PPCF128));
    4103             :     else
    4104          75 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
    4105             :                                         RTLIB::LOG10_F80, RTLIB::LOG10_F128,
    4106             :                                         RTLIB::LOG10_PPCF128));
    4107             :     break;
    4108          90 :   case ISD::FEXP:
    4109             :   case ISD::STRICT_FEXP:
    4110         101 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
    4111           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
    4112             :                                         RTLIB::EXP_FINITE_F64,
    4113             :                                         RTLIB::EXP_FINITE_F80,
    4114             :                                         RTLIB::EXP_FINITE_F128,
    4115             :                                         RTLIB::EXP_FINITE_PPCF128));
    4116             :     else
    4117          86 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
    4118             :                                         RTLIB::EXP_F80, RTLIB::EXP_F128,
    4119             :                                         RTLIB::EXP_PPCF128));
    4120             :     break;
    4121          77 :   case ISD::FEXP2:
    4122             :   case ISD::STRICT_FEXP2:
    4123          88 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
    4124           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
    4125             :                                         RTLIB::EXP2_FINITE_F64,
    4126             :                                         RTLIB::EXP2_FINITE_F80,
    4127             :                                         RTLIB::EXP2_FINITE_F128,
    4128             :                                         RTLIB::EXP2_FINITE_PPCF128));
    4129             :     else
    4130          73 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
    4131             :                                         RTLIB::EXP2_F80, RTLIB::EXP2_F128,
    4132             :                                         RTLIB::EXP2_PPCF128));
    4133             :     break;
    4134          36 :   case ISD::FTRUNC:
    4135          36 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
    4136             :                                       RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
    4137             :                                       RTLIB::TRUNC_PPCF128));
    4138          36 :     break;
    4139          69 :   case ISD::FFLOOR:
    4140          69 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
    4141             :                                       RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
    4142             :                                       RTLIB::FLOOR_PPCF128));
    4143          69 :     break;
    4144          26 :   case ISD::FCEIL:
    4145          26 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
    4146             :                                       RTLIB::CEIL_F80, RTLIB::CEIL_F128,
    4147             :                                       RTLIB::CEIL_PPCF128));
    4148          26 :     break;
    4149          33 :   case ISD::FRINT:
    4150             :   case ISD::STRICT_FRINT:
    4151          33 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
    4152             :                                       RTLIB::RINT_F80, RTLIB::RINT_F128,
    4153             :                                       RTLIB::RINT_PPCF128));
    4154          33 :     break;
    4155          50 :   case ISD::FNEARBYINT:
    4156             :   case ISD::STRICT_FNEARBYINT:
    4157          50 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
    4158             :                                       RTLIB::NEARBYINT_F64,
    4159             :                                       RTLIB::NEARBYINT_F80,
    4160             :                                       RTLIB::NEARBYINT_F128,
    4161             :                                       RTLIB::NEARBYINT_PPCF128));
    4162          50 :     break;
    4163          17 :   case ISD::FROUND:
    4164          17 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
    4165             :                                       RTLIB::ROUND_F64,
    4166             :                                       RTLIB::ROUND_F80,
    4167             :                                       RTLIB::ROUND_F128,
    4168             :                                       RTLIB::ROUND_PPCF128));
    4169          17 :     break;
    4170         110 :   case ISD::FPOWI:
    4171             :   case ISD::STRICT_FPOWI:
    4172         110 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
    4173             :                                       RTLIB::POWI_F80, RTLIB::POWI_F128,
    4174             :                                       RTLIB::POWI_PPCF128));
    4175         110 :     break;
    4176         107 :   case ISD::FPOW:
    4177             :   case ISD::STRICT_FPOW:
    4178         118 :     if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
    4179           4 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
    4180             :                                         RTLIB::POW_FINITE_F64,
    4181             :                                         RTLIB::POW_FINITE_F80,
    4182             :                                         RTLIB::POW_FINITE_F128,
    4183             :                                         RTLIB::POW_FINITE_PPCF128));
    4184             :     else
    4185         103 :       Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
    4186             :                                         RTLIB::POW_F80, RTLIB::POW_F128,
    4187             :                                         RTLIB::POW_PPCF128));
    4188             :     break;
    4189           9 :   case ISD::FDIV:
    4190           9 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
    4191             :                                       RTLIB::DIV_F80, RTLIB::DIV_F128,
    4192             :                                       RTLIB::DIV_PPCF128));
    4193           9 :     break;
    4194          91 :   case ISD::FREM:
    4195          91 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
    4196             :                                       RTLIB::REM_F80, RTLIB::REM_F128,
    4197             :                                       RTLIB::REM_PPCF128));
    4198          91 :     break;
    4199         233 :   case ISD::FMA:
    4200             :   case ISD::STRICT_FMA:
    4201         233 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
    4202             :                                       RTLIB::FMA_F80, RTLIB::FMA_F128,
    4203             :                                       RTLIB::FMA_PPCF128));
    4204         233 :     break;
    4205          43 :   case ISD::FADD:
    4206          43 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
    4207             :                                       RTLIB::ADD_F80, RTLIB::ADD_F128,
    4208             :                                       RTLIB::ADD_PPCF128));
    4209          43 :     break;
    4210         121 :   case ISD::FMUL:
    4211         121 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
    4212             :                                       RTLIB::MUL_F80, RTLIB::MUL_F128,
    4213             :                                       RTLIB::MUL_PPCF128));
    4214         121 :     break;
    4215             :   case ISD::FP16_TO_FP:
    4216         357 :     if (Node->getValueType(0) == MVT::f32) {
    4217         357 :       Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
    4218             :     }
    4219             :     break;
    4220             :   case ISD::FP_TO_FP16: {
    4221             :     RTLIB::Libcall LC =
    4222         932 :         RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
    4223             :     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
    4224         466 :     Results.push_back(ExpandLibCall(LC, Node, false));
    4225         466 :     break;
    4226             :   }
    4227          49 :   case ISD::FSUB:
    4228          49 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
    4229             :                                       RTLIB::SUB_F80, RTLIB::SUB_F128,
    4230             :                                       RTLIB::SUB_PPCF128));
    4231          49 :     break;
    4232          19 :   case ISD::SREM:
    4233          19 :     Results.push_back(ExpandIntLibCall(Node, true,
    4234             :                                        RTLIB::SREM_I8,
    4235             :                                        RTLIB::SREM_I16, RTLIB::SREM_I32,
    4236             :                                        RTLIB::SREM_I64, RTLIB::SREM_I128));
    4237          19 :     break;
    4238          13 :   case ISD::UREM:
    4239          13 :     Results.push_back(ExpandIntLibCall(Node, false,
    4240             :                                        RTLIB::UREM_I8,
    4241             :                                        RTLIB::UREM_I16, RTLIB::UREM_I32,
    4242             :                                        RTLIB::UREM_I64, RTLIB::UREM_I128));
    4243          13 :     break;
    4244          48 :   case ISD::SDIV:
    4245          48 :     Results.push_back(ExpandIntLibCall(Node, true,
    4246             :                                        RTLIB::SDIV_I8,
    4247             :                                        RTLIB::SDIV_I16, RTLIB::SDIV_I32,
    4248             :                                        RTLIB::SDIV_I64, RTLIB::SDIV_I128));
    4249          48 :     break;
    4250          21 :   case ISD::UDIV:
    4251          21 :     Results.push_back(ExpandIntLibCall(Node, false,
    4252             :                                        RTLIB::UDIV_I8,
    4253             :                                        RTLIB::UDIV_I16, RTLIB::UDIV_I32,
    4254             :                                        RTLIB::UDIV_I64, RTLIB::UDIV_I128));
    4255          21 :     break;
    4256          14 :   case ISD::SDIVREM:
    4257             :   case ISD::UDIVREM:
    4258             :     // Expand into divrem libcall
    4259          14 :     ExpandDivRemLibCall(Node, Results);
    4260          14 :     break;
    4261          15 :   case ISD::MUL:
    4262          15 :     Results.push_back(ExpandIntLibCall(Node, false,
    4263             :                                        RTLIB::MUL_I8,
    4264             :                                        RTLIB::MUL_I16, RTLIB::MUL_I32,
    4265             :                                        RTLIB::MUL_I64, RTLIB::MUL_I128));
    4266          15 :     break;
    4267             :   }
    4268             : 
    4269             :   // Replace the original node with the legalized result.
    4270       42049 :   if (!Results.empty()) {
    4271             :     LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
    4272        3033 :     ReplaceNode(Node, Results.data());
    4273             :   } else
    4274             :     LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
    4275       42049 : }
    4276             : 
    4277             : // Determine the vector type to use in place of an original scalar element when
    4278             : // promoting equally sized vectors.
    4279        3580 : static MVT getPromotedVectorElementType(const TargetLowering &TLI,
    4280             :                                         MVT EltVT, MVT NewEltVT) {
    4281        3580 :   unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
    4282        3580 :   MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
    4283             :   assert(TLI.isTypeLegal(MidVT) && "unexpected");
    4284        3580 :   return MidVT;
    4285             : }
    4286             : 
    4287        7227 : void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
    4288             :   LLVM_DEBUG(dbgs() << "Trying to promote node\n");
    4289             :   SmallVector<SDValue, 8> Results;
    4290        7227 :   MVT OVT = Node->getSimpleValueType(0);
    4291       14393 :   if (Node->getOpcode() == ISD::UINT_TO_FP ||
    4292        7134 :       Node->getOpcode() == ISD::SINT_TO_FP ||
    4293        6941 :       Node->getOpcode() == ISD::SETCC ||
    4294       11406 :       Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
    4295             :       Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
    4296        6104 :     OVT = Node->getOperand(0).getSimpleValueType();
    4297             :   }
    4298       14454 :   if (Node->getOpcode() == ISD::BR_CC)
    4299           2 :     OVT = Node->getOperand(2).getSimpleValueType();
    4300        7227 :   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
    4301             :   SDLoc dl(Node);
    4302        7227 :   SDValue Tmp1, Tmp2, Tmp3;
    4303       14454 :   switch (Node->getOpcode()) {
    4304          77 :   case ISD::CTTZ:
    4305             :   case ISD::CTTZ_ZERO_UNDEF:
    4306             :   case ISD::CTLZ:
    4307             :   case ISD::CTLZ_ZERO_UNDEF:
    4308             :   case ISD::CTPOP:
    4309             :     // Zero extend the argument.
    4310         231 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
    4311          77 :     if (Node->getOpcode() == ISD::CTTZ) {
    4312             :       // The count is the same in the promoted type except if the original
    4313             :       // value was zero.  This can be handled by setting the bit just off
    4314             :       // the top of the original type.
    4315             :       auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
    4316           3 :                                         OVT.getSizeInBits());
    4317           6 :       Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
    4318           6 :                          DAG.getConstant(TopBit, dl, NVT));
    4319             :     }
    4320             :     // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
    4321             :     // already the correct result.
    4322         231 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4323         154 :     if (Node->getOpcode() == ISD::CTLZ ||
    4324             :         Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
    4325             :       // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
    4326          22 :       Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
    4327          22 :                           DAG.getConstant(NVT.getSizeInBits() -
    4328          33 :                                           OVT.getSizeInBits(), dl, NVT));
    4329             :     }
    4330         154 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4331          77 :     break;
    4332           3 :   case ISD::BITREVERSE:
    4333             :   case ISD::BSWAP: {
    4334           3 :     unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
    4335           9 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
    4336           9 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4337           6 :     Tmp1 = DAG.getNode(
    4338             :         ISD::SRL, dl, NVT, Tmp1,
    4339             :         DAG.getConstant(DiffBits, dl,
    4340           9 :                         TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
    4341             : 
    4342           6 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4343           3 :     break;
    4344             :   }
    4345         142 :   case ISD::FP_TO_UINT:
    4346             :   case ISD::FP_TO_SINT:
    4347         284 :     Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
    4348         142 :                                  Node->getOpcode() == ISD::FP_TO_SINT, dl);
    4349         142 :     Results.push_back(Tmp1);
    4350         142 :     break;
    4351          93 :   case ISD::UINT_TO_FP:
    4352             :   case ISD::SINT_TO_FP:
    4353         186 :     Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
    4354          93 :                                  Node->getOpcode() == ISD::SINT_TO_FP, dl);
    4355          93 :     Results.push_back(Tmp1);
    4356          93 :     break;
    4357           1 :   case ISD::VAARG: {
    4358           1 :     SDValue Chain = Node->getOperand(0); // Get the chain.
    4359           1 :     SDValue Ptr = Node->getOperand(1); // Get the pointer.
    4360             : 
    4361             :     unsigned TruncOp;
    4362           1 :     if (OVT.isVector()) {
    4363             :       TruncOp = ISD::BITCAST;
    4364             :     } else {
    4365             :       assert(OVT.isInteger()
    4366             :         && "VAARG promotion is supported only for vectors or integer types");
    4367             :       TruncOp = ISD::TRUNCATE;
    4368             :     }
    4369             : 
    4370             :     // Perform the larger operation, then convert back
    4371           2 :     Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
    4372           2 :              Node->getConstantOperandVal(3));
    4373           1 :     Chain = Tmp1.getValue(1);
    4374             : 
    4375           2 :     Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
    4376             : 
    4377             :     // Modified the chain result - switch anything that used the old chain to
    4378             :     // use the new one.
    4379           2 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
    4380           2 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
    4381           1 :     if (UpdatedNodes) {
    4382           0 :       UpdatedNodes->insert(Tmp2.getNode());
    4383           0 :       UpdatedNodes->insert(Chain.getNode());
    4384             :     }
    4385           1 :     ReplacedNode(Node);
    4386             :     break;
    4387             :   }
    4388             :   case ISD::MUL:
    4389             :   case ISD::SDIV:
    4390             :   case ISD::SREM:
    4391             :   case ISD::UDIV:
    4392             :   case ISD::UREM:
    4393             :   case ISD::AND:
    4394             :   case ISD::OR:
    4395             :   case ISD::XOR: {
    4396             :     unsigned ExtOp, TruncOp;
    4397        1922 :     if (OVT.isVector()) {
    4398             :       ExtOp   = ISD::BITCAST;
    4399             :       TruncOp = ISD::BITCAST;
    4400             :     } else {
    4401             :       assert(OVT.isInteger() && "Cannot promote logic operation");
    4402             : 
    4403          11 :       switch (Node->getOpcode()) {
    4404             :       default:
    4405             :         ExtOp = ISD::ANY_EXTEND;
    4406             :         break;
    4407           4 :       case ISD::SDIV:
    4408             :       case ISD::SREM:
    4409             :         ExtOp = ISD::SIGN_EXTEND;
    4410           4 :         break;
    4411           6 :       case ISD::UDIV:
    4412             :       case ISD::UREM:
    4413             :         ExtOp = ISD::ZERO_EXTEND;
    4414           6 :         break;
    4415             :       }
    4416             :       TruncOp = ISD::TRUNCATE;
    4417             :     }
    4418             :     // Promote each of the values to the new type.
    4419        5766 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4420        5766 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4421             :     // Perform the larger operation, then convert back
    4422        5766 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
    4423        3844 :     Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
    4424        1922 :     break;
    4425             :   }
    4426           0 :   case ISD::UMUL_LOHI:
    4427             :   case ISD::SMUL_LOHI: {
    4428             :     // Promote to a multiply in a wider integer type.
    4429           0 :     unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
    4430             :                                                          : ISD::SIGN_EXTEND;
    4431           0 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4432           0 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4433           0 :     Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
    4434             : 
    4435           0 :     auto &DL = DAG.getDataLayout();
    4436           0 :     unsigned OriginalSize = OVT.getScalarSizeInBits();
    4437           0 :     Tmp2 = DAG.getNode(
    4438             :         ISD::SRL, dl, NVT, Tmp1,
    4439           0 :         DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
    4440           0 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4441           0 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
    4442           0 :     break;
    4443             :   }
    4444         164 :   case ISD::SELECT: {
    4445             :     unsigned ExtOp, TruncOp;
    4446         820 :     if (Node->getValueType(0).isVector() ||
    4447         328 :         Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
    4448             :       ExtOp   = ISD::BITCAST;
    4449             :       TruncOp = ISD::BITCAST;
    4450           8 :     } else if (Node->getValueType(0).isInteger()) {
    4451             :       ExtOp   = ISD::ANY_EXTEND;
    4452             :       TruncOp = ISD::TRUNCATE;
    4453             :     } else {
    4454             :       ExtOp   = ISD::FP_EXTEND;
    4455             :       TruncOp = ISD::FP_ROUND;
    4456             :     }
    4457         164 :     Tmp1 = Node->getOperand(0);
    4458             :     // Promote each of the values to the new type.
    4459         328 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4460         492 :     Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
    4461             :     // Perform the larger operation, then round down.
    4462         328 :     Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
    4463         164 :     if (TruncOp != ISD::FP_ROUND)
    4464         480 :       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
    4465             :     else
    4466           8 :       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
    4467           8 :                          DAG.getIntPtrConstant(0, dl));
    4468         164 :     Results.push_back(Tmp1);
    4469         164 :     break;
    4470             :   }
    4471             :   case ISD::VECTOR_SHUFFLE: {
    4472         509 :     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
    4473             : 
    4474             :     // Cast the two input vectors.
    4475        1527 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
    4476        1527 :     Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
    4477             : 
    4478             :     // Convert the shuffle mask to the right # elements.
    4479         509 :     Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
    4480        1018 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
    4481         509 :     Results.push_back(Tmp1);
    4482             :     break;
    4483             :   }
    4484         193 :   case ISD::SETCC: {
    4485             :     unsigned ExtOp = ISD::FP_EXTEND;
    4486         193 :     if (NVT.isInteger()) {
    4487             :       ISD::CondCode CCCode =
    4488         152 :         cast<CondCodeSDNode>(Node->getOperand(2))->get();
    4489          76 :       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    4490             :     }
    4491         579 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4492         579 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4493         386 :     Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
    4494         579 :                                   Tmp1, Tmp2, Node->getOperand(2)));
    4495         193 :     break;
    4496             :   }
    4497           1 :   case ISD::BR_CC: {
    4498             :     unsigned ExtOp = ISD::FP_EXTEND;
    4499           1 :     if (NVT.isInteger()) {
    4500             :       ISD::CondCode CCCode =
    4501           0 :         cast<CondCodeSDNode>(Node->getOperand(1))->get();
    4502           0 :       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    4503             :     }
    4504           3 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
    4505           3 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
    4506           2 :     Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
    4507             :                                   Node->getOperand(0), Node->getOperand(1),
    4508           2 :                                   Tmp1, Tmp2, Node->getOperand(4)));
    4509           1 :     break;
    4510             :   }
    4511         190 :   case ISD::FADD:
    4512             :   case ISD::FSUB:
    4513             :   case ISD::FMUL:
    4514             :   case ISD::FDIV:
    4515             :   case ISD::FREM:
    4516             :   case ISD::FMINNUM:
    4517             :   case ISD::FMAXNUM:
    4518             :   case ISD::FPOW:
    4519         570 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4520         570 :     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
    4521         380 :     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
    4522         380 :                        Node->getFlags());
    4523         380 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4524         380 :                                   Tmp3, DAG.getIntPtrConstant(0, dl)));
    4525         190 :     break;
    4526          25 :   case ISD::FMA:
    4527          75 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4528          75 :     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
    4529          75 :     Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
    4530          25 :     Results.push_back(
    4531          75 :         DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4532          25 :                     DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
    4533         100 :                     DAG.getIntPtrConstant(0, dl)));
    4534          25 :     break;
    4535          12 :   case ISD::FCOPYSIGN:
    4536             :   case ISD::FPOWI: {
    4537          36 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4538          12 :     Tmp2 = Node->getOperand(1);
    4539          36 :     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
    4540             : 
    4541             :     // fcopysign doesn't change anything but the sign bit, so
    4542             :     //   (fp_round (fcopysign (fpext a), b))
    4543             :     // is as precise as
    4544             :     //   (fp_round (fpext a))
    4545             :     // which is a no-op. Mark it as a TRUNCating FP_ROUND.
    4546          12 :     const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
    4547          24 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4548          24 :                                   Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
    4549          12 :     break;
    4550             :   }
    4551         163 :   case ISD::FFLOOR:
    4552             :   case ISD::FCEIL:
    4553             :   case ISD::FRINT:
    4554             :   case ISD::FNEARBYINT:
    4555             :   case ISD::FROUND:
    4556             :   case ISD::FTRUNC:
    4557             :   case ISD::FNEG:
    4558             :   case ISD::FSQRT:
    4559             :   case ISD::FSIN:
    4560             :   case ISD::FCOS:
    4561             :   case ISD::FLOG:
    4562             :   case ISD::FLOG2:
    4563             :   case ISD::FLOG10:
    4564             :   case ISD::FABS:
    4565             :   case ISD::FEXP:
    4566             :   case ISD::FEXP2:
    4567         489 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4568         489 :     Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4569         326 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4570         326 :                                   Tmp2, DAG.getIntPtrConstant(0, dl)));
    4571         163 :     break;
    4572         411 :   case ISD::BUILD_VECTOR: {
    4573         411 :     MVT EltVT = OVT.getVectorElementType();
    4574         411 :     MVT NewEltVT = NVT.getVectorElementType();
    4575             : 
    4576             :     // Handle bitcasts to a different vector type with the same total bit size
    4577             :     //
    4578             :     // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
    4579             :     //  =>
    4580             :     //  v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
    4581             : 
    4582             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4583             :            "Invalid promote type for build_vector");
    4584             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4585             : 
    4586         411 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4587             : 
    4588             :     SmallVector<SDValue, 8> NewOps;
    4589        2466 :     for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
    4590        1644 :       SDValue Op = Node->getOperand(I);
    4591        2466 :       NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
    4592             :     }
    4593             : 
    4594             :     SDLoc SL(Node);
    4595         822 :     SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
    4596         822 :     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
    4597         411 :     Results.push_back(CvtVec);
    4598             :     break;
    4599             :   }
    4600        2762 :   case ISD::EXTRACT_VECTOR_ELT: {
    4601        2762 :     MVT EltVT = OVT.getVectorElementType();
    4602        2762 :     MVT NewEltVT = NVT.getVectorElementType();
    4603             : 
    4604             :     // Handle bitcasts to a different vector type with the same total bit size.
    4605             :     //
    4606             :     // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
    4607             :     //  =>
    4608             :     //  v4i32:castx = bitcast x:v2i64
    4609             :     //
    4610             :     // i64 = bitcast
    4611             :     //   (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
    4612             :     //                       (i32 (extract_vector_elt castx, (2 * y + 1)))
    4613             :     //
    4614             : 
    4615             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4616             :            "Invalid promote type for extract_vector_elt");
    4617             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4618             : 
    4619        2762 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4620        2762 :     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
    4621             : 
    4622        2762 :     SDValue Idx = Node->getOperand(1);
    4623        2762 :     EVT IdxVT = Idx.getValueType();
    4624             :     SDLoc SL(Node);
    4625        2762 :     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
    4626        5524 :     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
    4627             : 
    4628        8286 :     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
    4629             : 
    4630             :     SmallVector<SDValue, 8> NewOps;
    4631       13810 :     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
    4632        5524 :       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
    4633       11048 :       SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
    4634             : 
    4635        5524 :       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
    4636        5524 :                                 CastVec, TmpIdx);
    4637        5524 :       NewOps.push_back(Elt);
    4638             :     }
    4639             : 
    4640        5524 :     SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
    4641        5524 :     Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
    4642             :     break;
    4643             :   }
    4644           4 :   case ISD::INSERT_VECTOR_ELT: {
    4645           4 :     MVT EltVT = OVT.getVectorElementType();
    4646           4 :     MVT NewEltVT = NVT.getVectorElementType();
    4647             : 
    4648             :     // Handle bitcasts to a different vector type with the same total bit size
    4649             :     //
    4650             :     // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
    4651             :     //  =>
    4652             :     //  v4i32:castx = bitcast x:v2i64
    4653             :     //  v2i32:casty = bitcast y:i64
    4654             :     //
    4655             :     // v2i64 = bitcast
    4656             :     //   (v4i32 insert_vector_elt
    4657             :     //       (v4i32 insert_vector_elt v4i32:castx,
    4658             :     //                                (extract_vector_elt casty, 0), 2 * z),
    4659             :     //        (extract_vector_elt casty, 1), (2 * z + 1))
    4660             : 
    4661             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4662             :            "Invalid promote type for insert_vector_elt");
    4663             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4664             : 
    4665           4 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4666           4 :     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
    4667             : 
    4668           4 :     SDValue Val = Node->getOperand(1);
    4669           4 :     SDValue Idx = Node->getOperand(2);
    4670           4 :     EVT IdxVT = Idx.getValueType();
    4671             :     SDLoc SL(Node);
    4672             : 
    4673           8 :     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
    4674           8 :     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
    4675             : 
    4676          12 :     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
    4677           8 :     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
    4678             : 
    4679           4 :     SDValue NewVec = CastVec;
    4680          20 :     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
    4681           8 :       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
    4682          16 :       SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
    4683             : 
    4684           8 :       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
    4685           8 :                                 CastVal, IdxOffset);
    4686             : 
    4687          16 :       NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
    4688           8 :                            NewVec, Elt, InEltIdx);
    4689             :     }
    4690             : 
    4691           8 :     Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
    4692             :     break;
    4693             :   }
    4694         403 :   case ISD::SCALAR_TO_VECTOR: {
    4695         403 :     MVT EltVT = OVT.getVectorElementType();
    4696         403 :     MVT NewEltVT = NVT.getVectorElementType();
    4697             : 
    4698             :     // Handle bitcasts to different vector type with the same total bit size.
    4699             :     //
    4700             :     // e.g. v2i64 = scalar_to_vector x:i64
    4701             :     //   =>
    4702             :     //  concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
    4703             :     //
    4704             : 
    4705         403 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4706         403 :     SDValue Val = Node->getOperand(0);
    4707             :     SDLoc SL(Node);
    4708             : 
    4709         806 :     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
    4710         806 :     SDValue Undef = DAG.getUNDEF(MidVT);
    4711             : 
    4712             :     SmallVector<SDValue, 8> NewElts;
    4713         403 :     NewElts.push_back(CastVal);
    4714         806 :     for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
    4715         403 :       NewElts.push_back(Undef);
    4716             : 
    4717         806 :     SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
    4718         806 :     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
    4719         403 :     Results.push_back(CvtVec);
    4720             :     break;
    4721             :   }
    4722             :   }
    4723             : 
    4724             :   // Replace the original node with the legalized result.
    4725        7227 :   if (!Results.empty()) {
    4726             :     LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
    4727        7074 :     ReplaceNode(Node, Results.data());
    4728             :   } else
    4729             :     LLVM_DEBUG(dbgs() << "Could not promote node\n");
    4730        7227 : }
    4731             : 
    4732             : /// This is the entry point for the file.
    4733      364595 : void SelectionDAG::Legalize() {
    4734      364595 :   AssignTopologicalOrder();
    4735             : 
    4736             :   SmallPtrSet<SDNode *, 16> LegalizedNodes;
    4737             :   // Use a delete listener to remove nodes which were deleted during
    4738             :   // legalization from LegalizeNodes. This is needed to handle the situation
    4739             :   // where a new node is allocated by the object pool to the same address of a
    4740             :   // previously deleted node.
    4741             :   DAGNodeDeletedListener DeleteListener(
    4742             :       *this,
    4743      364942 :       [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
    4744             : 
    4745             :   SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
    4746             : 
    4747             :   // Visit all the nodes. We start in topological order, so that we see
    4748             :   // nodes with their original operands intact. Legalization can produce
    4749             :   // new nodes which may themselves need to be legalized. Iterate until all
    4750             :   // nodes have been legalized.
    4751             :   while (true) {
    4752             :     bool AnyLegalized = false;
    4753    32229232 :     for (auto NI = allnodes_end(); NI != allnodes_begin();) {
    4754             :       --NI;
    4755             : 
    4756             :       SDNode *N = &*NI;
    4757    31734022 :       if (N->use_empty() && N != getRoot().getNode()) {
    4758             :         ++NI;
    4759      530818 :         DeleteNode(N);
    4760      530818 :         continue;
    4761             :       }
    4762             : 
    4763    30672386 :       if (LegalizedNodes.insert(N).second) {
    4764             :         AnyLegalized = true;
    4765    11809245 :         Legalizer.LegalizeOp(N);
    4766             : 
    4767    11809244 :         if (N->use_empty() && N != getRoot().getNode()) {
    4768             :           ++NI;
    4769     1133579 :           DeleteNode(N);
    4770             :         }
    4771             :       }
    4772             :     }
    4773     1026028 :     if (!AnyLegalized)
    4774             :       break;
    4775             : 
    4776             :   }
    4777             : 
    4778             :   // Remove dead nodes now.
    4779      364594 :   RemoveDeadNodes();
    4780      364594 : }
    4781             : 
    4782    11673900 : bool SelectionDAG::LegalizeOp(SDNode *N,
    4783             :                               SmallSetVector<SDNode *, 16> &UpdatedNodes) {
    4784             :   SmallPtrSet<SDNode *, 16> LegalizedNodes;
    4785             :   SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
    4786             : 
    4787             :   // Directly insert the node in question, and legalize it. This will recurse
    4788             :   // as needed through operands.
    4789    11673900 :   LegalizedNodes.insert(N);
    4790    11673900 :   Legalizer.LegalizeOp(N);
    4791             : 
    4792    23347800 :   return LegalizedNodes.count(N);
    4793             : }

Generated by: LCOV version 1.13