LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeDAG.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 2187 2478 88.3 %
Date: 2017-09-14 15:23:50 Functions: 45 47 95.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the SelectionDAG::Legalize method.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/ADT/SetVector.h"
      15             : #include "llvm/ADT/SmallPtrSet.h"
      16             : #include "llvm/ADT/SmallSet.h"
      17             : #include "llvm/ADT/SmallVector.h"
      18             : #include "llvm/ADT/Triple.h"
      19             : #include "llvm/CodeGen/MachineFunction.h"
      20             : #include "llvm/CodeGen/MachineJumpTableInfo.h"
      21             : #include "llvm/CodeGen/SelectionDAG.h"
      22             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      23             : #include "llvm/IR/CallingConv.h"
      24             : #include "llvm/IR/Constants.h"
      25             : #include "llvm/IR/DataLayout.h"
      26             : #include "llvm/IR/DebugInfo.h"
      27             : #include "llvm/IR/DerivedTypes.h"
      28             : #include "llvm/IR/Function.h"
      29             : #include "llvm/IR/LLVMContext.h"
      30             : #include "llvm/Support/Debug.h"
      31             : #include "llvm/Support/ErrorHandling.h"
      32             : #include "llvm/Support/MathExtras.h"
      33             : #include "llvm/Support/raw_ostream.h"
      34             : #include "llvm/Target/TargetFrameLowering.h"
      35             : #include "llvm/Target/TargetLowering.h"
      36             : #include "llvm/Target/TargetMachine.h"
      37             : #include "llvm/Target/TargetSubtargetInfo.h"
      38             : using namespace llvm;
      39             : 
      40             : #define DEBUG_TYPE "legalizedag"
      41             : 
      42             : namespace {
      43             : 
      44             : struct FloatSignAsInt;
      45             : 
      46             : //===----------------------------------------------------------------------===//
      47             : /// This takes an arbitrary SelectionDAG as input and
      48             : /// hacks on it until the target machine can handle it.  This involves
      49             : /// eliminating value sizes the machine cannot handle (promoting small sizes to
      50             : /// large sizes or splitting up large values into small values) as well as
      51             : /// eliminating operations the machine cannot handle.
      52             : ///
      53             : /// This code also does a small amount of optimization and recognition of idioms
      54             : /// as part of its processing.  For example, if a target does not support a
      55             : /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
      56             : /// will attempt merge setcc and brc instructions into brcc's.
      57             : ///
      58             : class SelectionDAGLegalize {
      59             :   const TargetMachine &TM;
      60             :   const TargetLowering &TLI;
      61             :   SelectionDAG &DAG;
      62             : 
      63             :   /// \brief The set of nodes which have already been legalized. We hold a
      64             :   /// reference to it in order to update as necessary on node deletion.
      65             :   SmallPtrSetImpl<SDNode *> &LegalizedNodes;
      66             : 
      67             :   /// \brief A set of all the nodes updated during legalization.
      68             :   SmallSetVector<SDNode *, 16> *UpdatedNodes;
      69             : 
      70         412 :   EVT getSetCCResultType(EVT VT) const {
      71         824 :     return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
      72             :   }
      73             : 
      74             :   // Libcall insertion helpers.
      75             : 
      76             : public:
      77             :   SelectionDAGLegalize(SelectionDAG &DAG,
      78             :                        SmallPtrSetImpl<SDNode *> &LegalizedNodes,
      79             :                        SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
      80    21302656 :       : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
      81    21302656 :         LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
      82             : 
      83             :   /// \brief Legalizes the given operation.
      84             :   void LegalizeOp(SDNode *Node);
      85             : 
      86             : private:
      87             :   SDValue OptimizeFloatStore(StoreSDNode *ST);
      88             : 
      89             :   void LegalizeLoadOps(SDNode *Node);
      90             :   void LegalizeStoreOps(SDNode *Node);
      91             : 
      92             :   /// Some targets cannot handle a variable
      93             :   /// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
      94             :   /// is necessary to spill the vector being inserted into to memory, perform
      95             :   /// the insert there, and then read the result back.
      96             :   SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
      97             :                                          const SDLoc &dl);
      98             :   SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
      99             :                                   const SDLoc &dl);
     100             : 
     101             :   /// Return a vector shuffle operation which
     102             :   /// performs the same shuffe in terms of order or result bytes, but on a type
     103             :   /// whose vector element type is narrower than the original shuffle type.
     104             :   /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
     105             :   SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
     106             :                                      SDValue N1, SDValue N2,
     107             :                                      ArrayRef<int> Mask) const;
     108             : 
     109             :   bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
     110             :                              bool &NeedInvert, const SDLoc &dl);
     111             : 
     112             :   SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
     113             :   SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
     114             :                         unsigned NumOps, bool isSigned, const SDLoc &dl);
     115             : 
     116             :   std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
     117             :                                                  SDNode *Node, bool isSigned);
     118             :   SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
     119             :                           RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
     120             :                           RTLIB::Libcall Call_F128,
     121             :                           RTLIB::Libcall Call_PPCF128);
     122             :   SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
     123             :                            RTLIB::Libcall Call_I8,
     124             :                            RTLIB::Libcall Call_I16,
     125             :                            RTLIB::Libcall Call_I32,
     126             :                            RTLIB::Libcall Call_I64,
     127             :                            RTLIB::Libcall Call_I128);
     128             :   void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
     129             :   void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
     130             : 
     131             :   SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
     132             :                            const SDLoc &dl);
     133             :   SDValue ExpandBUILD_VECTOR(SDNode *Node);
     134             :   SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
     135             :   void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
     136             :                                 SmallVectorImpl<SDValue> &Results);
     137             :   void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
     138             :                          SDValue Value) const;
     139             :   SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
     140             :                           SDValue NewIntValue) const;
     141             :   SDValue ExpandFCOPYSIGN(SDNode *Node) const;
     142             :   SDValue ExpandFABS(SDNode *Node) const;
     143             :   SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
     144             :                                const SDLoc &dl);
     145             :   SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
     146             :                                 const SDLoc &dl);
     147             :   SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
     148             :                                 const SDLoc &dl);
     149             : 
     150             :   SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
     151             :   SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
     152             :   SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
     153             : 
     154             :   SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
     155             :   SDValue ExpandInsertToVectorThroughStack(SDValue Op);
     156             :   SDValue ExpandVectorBuildThroughStack(SDNode* Node);
     157             : 
     158             :   SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
     159             :   SDValue ExpandConstant(ConstantSDNode *CP);
     160             : 
     161             :   // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
     162             :   bool ExpandNode(SDNode *Node);
     163             :   void ConvertNodeToLibcall(SDNode *Node);
     164             :   void PromoteNode(SDNode *Node);
     165             : 
     166             : public:
     167             :   // Node replacement helpers
     168     1055647 :   void ReplacedNode(SDNode *N) {
     169     2111294 :     LegalizedNodes.erase(N);
     170     1055647 :     if (UpdatedNodes)
     171        7144 :       UpdatedNodes->insert(N);
     172     1055647 :   }
     173         118 :   void ReplaceNode(SDNode *Old, SDNode *New) {
     174             :     DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
     175             :           dbgs() << "     with:      "; New->dump(&DAG));
     176             : 
     177             :     assert(Old->getNumValues() == New->getNumValues() &&
     178             :            "Replacing one node with another that produces a different number "
     179             :            "of values!");
     180         118 :     DAG.ReplaceAllUsesWith(Old, New);
     181         118 :     if (UpdatedNodes)
     182           1 :       UpdatedNodes->insert(New);
     183         118 :     ReplacedNode(Old);
     184         118 :   }
     185      664646 :   void ReplaceNode(SDValue Old, SDValue New) {
     186             :     DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
     187             :           dbgs() << "     with:      "; New->dump(&DAG));
     188             : 
     189      664646 :     DAG.ReplaceAllUsesWith(Old, New);
     190      664646 :     if (UpdatedNodes)
     191        3471 :       UpdatedNodes->insert(New.getNode());
     192      664646 :     ReplacedNode(Old.getNode());
     193      664646 :   }
     194      320663 :   void ReplaceNode(SDNode *Old, const SDValue *New) {
     195             :     DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
     196             : 
     197      320663 :     DAG.ReplaceAllUsesWith(Old, New);
     198     1241534 :     for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
     199             :       DEBUG(dbgs() << (i == 0 ? "     with:      "
     200             :                               : "      and:      ");
     201             :             New[i]->dump(&DAG));
     202      600208 :       if (UpdatedNodes)
     203        1980 :         UpdatedNodes->insert(New[i].getNode());
     204             :     }
     205      320663 :     ReplacedNode(Old);
     206      320663 :   }
     207             : };
     208             : }
     209             : 
     210             : /// Return a vector shuffle operation which
     211             : /// performs the same shuffe in terms of order or result bytes, but on a type
     212             : /// whose vector element type is narrower than the original shuffle type.
     213             : /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
     214         402 : SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
     215             :     EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
     216             :     ArrayRef<int> Mask) const {
     217         402 :   unsigned NumMaskElts = VT.getVectorNumElements();
     218         402 :   unsigned NumDestElts = NVT.getVectorNumElements();
     219         402 :   unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
     220             : 
     221             :   assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
     222             : 
     223         402 :   if (NumEltsGrowth == 1)
     224           0 :     return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
     225             : 
     226         402 :   SmallVector<int, 8> NewMask;
     227        2354 :   for (unsigned i = 0; i != NumMaskElts; ++i) {
     228        3904 :     int Idx = Mask[i];
     229        8384 :     for (unsigned j = 0; j != NumEltsGrowth; ++j) {
     230        6432 :       if (Idx < 0)
     231         712 :         NewMask.push_back(-1);
     232             :       else
     233        5720 :         NewMask.push_back(Idx * NumEltsGrowth + j);
     234             :     }
     235             :   }
     236             :   assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
     237             :   assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
     238         804 :   return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
     239             : }
     240             : 
     241             : /// Expands the ConstantFP node to an integer constant or
     242             : /// a load from the constant pool.
     243             : SDValue
     244        2786 : SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
     245        2786 :   bool Extend = false;
     246        5572 :   SDLoc dl(CFP);
     247             : 
     248             :   // If a FP immediate is precise when represented as a float and if the
     249             :   // target can do an extending load from float to double, we put it into
     250             :   // the constant pool as a float, even if it's is statically typed as a
     251             :   // double.  This shrinks FP constants and canonicalizes them for targets where
     252             :   // an FP extending load is the same cost as a normal load (such as on the x87
     253             :   // fp stack or PPC FP unit).
     254        5572 :   EVT VT = CFP->getValueType(0);
     255        2786 :   ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
     256        2786 :   if (!UseCP) {
     257             :     assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
     258           0 :     return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
     259           0 :                            (VT == MVT::f64) ? MVT::i64 : MVT::i32);
     260             :   }
     261             : 
     262        8358 :   APFloat APF = CFP->getValueAPF();
     263        2786 :   EVT OrigVT = VT;
     264        2786 :   EVT SVT = VT;
     265             : 
     266             :   // We don't want to shrink SNaNs. Converting the SNaN back to its real type
     267             :   // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
     268        2786 :   if (!APF.isSignaling()) {
     269        5801 :     while (SVT != MVT::f32 && SVT != MVT::f16) {
     270        3000 :       SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
     271        2634 :       if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
     272             :           // Only do this if the target has a native EXTLOAD instruction from
     273             :           // smaller type.
     274        3481 :           TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
     275         847 :           TLI.ShouldShrinkFPConstant(OrigVT)) {
     276         485 :         Type *SType = SVT.getTypeForEVT(*DAG.getContext());
     277         970 :         LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
     278         485 :         VT = SVT;
     279         485 :         Extend = true;
     280             :       }
     281             :     }
     282             :   }
     283             : 
     284             :   SDValue CPIdx =
     285       11144 :       DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
     286        2786 :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
     287        2786 :   if (Extend) {
     288         409 :     SDValue Result = DAG.getExtLoad(
     289         409 :         ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
     290             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
     291         818 :         Alignment);
     292         409 :     return Result;
     293             :   }
     294        2377 :   SDValue Result = DAG.getLoad(
     295        2377 :       OrigVT, dl, DAG.getEntryNode(), CPIdx,
     296        4754 :       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
     297        2377 :   return Result;
     298             : }
     299             : 
     300             : /// Expands the Constant node to a load from the constant pool.
     301           0 : SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
     302           0 :   SDLoc dl(CP);
     303           0 :   EVT VT = CP->getValueType(0);
     304           0 :   SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
     305           0 :                                       TLI.getPointerTy(DAG.getDataLayout()));
     306           0 :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
     307           0 :   SDValue Result = DAG.getLoad(
     308           0 :       VT, dl, DAG.getEntryNode(), CPIdx,
     309           0 :       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
     310           0 :   return Result;
     311             : }
     312             : 
     313             : /// Some target cannot handle a variable insertion index for the
     314             : /// INSERT_VECTOR_ELT instruction.  In this case, it
     315             : /// is necessary to spill the vector being inserted into to memory, perform
     316             : /// the insert there, and then read the result back.
     317          16 : SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
     318             :                                                              SDValue Val,
     319             :                                                              SDValue Idx,
     320             :                                                              const SDLoc &dl) {
     321          16 :   SDValue Tmp1 = Vec;
     322          16 :   SDValue Tmp2 = Val;
     323          16 :   SDValue Tmp3 = Idx;
     324             : 
     325             :   // If the target doesn't support this, we have to spill the input vector
     326             :   // to a temporary stack slot, update the element, then reload it.  This is
     327             :   // badness.  We could also load the value into a vector register (either
     328             :   // with a "move to register" or "extload into register" instruction, then
     329             :   // permute it into place, if the idx is a constant and if the idx is
     330             :   // supported by the target.
     331          32 :   EVT VT    = Tmp1.getValueType();
     332          16 :   EVT EltVT = VT.getVectorElementType();
     333          16 :   SDValue StackPtr = DAG.CreateStackTemporary(VT);
     334             : 
     335          32 :   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
     336             : 
     337             :   // Store the vector.
     338          16 :   SDValue Ch = DAG.getStore(
     339          16 :       DAG.getEntryNode(), dl, Tmp1, StackPtr,
     340          32 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
     341             : 
     342          16 :   SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
     343             : 
     344             :   // Store the scalar value.
     345          48 :   Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
     346             :   // Load the updated vector.
     347          16 :   return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
     348          16 :                                                DAG.getMachineFunction(), SPFI));
     349             : }
     350             : 
     351         386 : SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
     352             :                                                       SDValue Idx,
     353             :                                                       const SDLoc &dl) {
     354         370 :   if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
     355             :     // SCALAR_TO_VECTOR requires that the type of the value being inserted
     356             :     // match the element type of the vector being created, except for
     357             :     // integers in which case the inserted value can be over width.
     358         740 :     EVT EltVT = Vec.getValueType().getVectorElementType();
     359         741 :     if (Val.getValueType() == EltVT ||
     360           3 :         (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
     361         370 :       SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
     362         740 :                                   Vec.getValueType(), Val);
     363             : 
     364         740 :       unsigned NumElts = Vec.getValueType().getVectorNumElements();
     365             :       // We generate a shuffle of InVec and ScVec, so the shuffle mask
     366             :       // should be 0,1,2,3,4,5... with the appropriate element replaced with
     367             :       // elt 0 of the RHS.
     368         740 :       SmallVector<int, 8> ShufOps;
     369        3860 :       for (unsigned i = 0; i != NumElts; ++i)
     370        3120 :         ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
     371             : 
     372        1110 :       return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
     373             :     }
     374             :   }
     375          16 :   return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
     376             : }
     377             : 
     378     1436504 : SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
     379             :   // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
     380             :   // FIXME: We shouldn't do this for TargetConstantFP's.
     381             :   // FIXME: move this to the DAG Combiner!  Note that we can't regress due
     382             :   // to phase ordering between legalized code and the dag combiner.  This
     383             :   // probably means that we need to integrate dag combiner and legalizer
     384             :   // together.
     385             :   // We generally can't do this one for long doubles.
     386     2873008 :   SDValue Chain = ST->getChain();
     387     1436504 :   SDValue Ptr = ST->getBasePtr();
     388     2873008 :   unsigned Alignment = ST->getAlignment();
     389     1436504 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
     390     2873008 :   AAMDNodes AAInfo = ST->getAAInfo();
     391     2873008 :   SDLoc dl(ST);
     392     1436676 :   if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
     393         344 :     if (CFP->getValueType(0) == MVT::f32 &&
     394           3 :         TLI.isTypeLegal(MVT::i32)) {
     395           1 :       SDValue Con = DAG.getConstant(CFP->getValueAPF().
     396           3 :                                       bitcastToAPInt().zextOrTrunc(32),
     397           4 :                                     SDLoc(CFP), MVT::i32);
     398           2 :       return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
     399           2 :                           MMOFlags, AAInfo);
     400             :     }
     401             : 
     402         334 :     if (CFP->getValueType(0) == MVT::f64) {
     403             :       // If this target supports 64-bit registers, do a single 64-bit store.
     404          24 :       if (TLI.isTypeLegal(MVT::i64)) {
     405           0 :         SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
     406           0 :                                       zextOrTrunc(64), SDLoc(CFP), MVT::i64);
     407           0 :         return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
     408           0 :                             Alignment, MMOFlags, AAInfo);
     409             :       }
     410             : 
     411          32 :       if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
     412             :         // Otherwise, if the target supports 32-bit registers, use 2 32-bit
     413             :         // stores.  If the target supports neither 32- nor 64-bits, this
     414             :         // xform is certainly not worth it.
     415           0 :         const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
     416           0 :         SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
     417           0 :         SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
     418           0 :         if (DAG.getDataLayout().isBigEndian())
     419             :           std::swap(Lo, Hi);
     420             : 
     421           0 :         Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
     422           0 :                           MMOFlags, AAInfo);
     423           0 :         Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     424           0 :                           DAG.getConstant(4, dl, Ptr.getValueType()));
     425           0 :         Hi = DAG.getStore(Chain, dl, Hi, Ptr,
     426           0 :                           ST->getPointerInfo().getWithOffset(4),
     427           0 :                           MinAlign(Alignment, 4U), MMOFlags, AAInfo);
     428             : 
     429           0 :         return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
     430             :       }
     431             :     }
     432             :   }
     433     1436503 :   return SDValue(nullptr, 0);
     434             : }
     435             : 
     436     1466538 : void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
     437     1466538 :     StoreSDNode *ST = cast<StoreSDNode>(Node);
     438     2933076 :     SDValue Chain = ST->getChain();
     439     1466538 :     SDValue Ptr = ST->getBasePtr();
     440     1494933 :     SDLoc dl(Node);
     441             : 
     442     2933076 :     unsigned Alignment = ST->getAlignment();
     443     1466538 :     MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
     444     1466538 :     AAMDNodes AAInfo = ST->getAAInfo();
     445             : 
     446     1466538 :     if (!ST->isTruncatingStore()) {
     447     1436504 :       if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
     448           1 :         ReplaceNode(ST, OptStore);
     449     1438144 :         return;
     450             :       }
     451             : 
     452             :       {
     453     1436503 :         SDValue Value = ST->getValue();
     454     1436503 :         MVT VT = Value.getSimpleValueType();
     455     4309509 :         switch (TLI.getOperationAction(ISD::STORE, VT)) {
     456           0 :         default: llvm_unreachable("This action is not supported yet!");
     457     1327081 :         case TargetLowering::Legal: {
     458             :           // If this is an unaligned store and the target doesn't support it,
     459             :           // expand it.
     460     1327081 :           EVT MemVT = ST->getMemoryVT();
     461     2654162 :           unsigned AS = ST->getAddressSpace();
     462     2654162 :           unsigned Align = ST->getAlignment();
     463     2654162 :           const DataLayout &DL = DAG.getDataLayout();
     464     1327081 :           if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     465         222 :             SDValue Result = TLI.expandUnalignedStore(ST, DAG);
     466         222 :             ReplaceNode(SDValue(ST, 0), Result);
     467             :           }
     468             :           break;
     469             :         }
     470      100393 :         case TargetLowering::Custom: {
     471      200786 :           SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
     472      113164 :           if (Res && Res != SDValue(Node, 0))
     473       12771 :             ReplaceNode(SDValue(Node, 0), Res);
     474             :           return;
     475             :         }
     476        9029 :         case TargetLowering::Promote: {
     477        9029 :           MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
     478             :           assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
     479             :                  "Can only promote stores to same size type");
     480       27087 :           Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
     481             :           SDValue Result =
     482       18058 :               DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     483       18058 :                            Alignment, MMOFlags, AAInfo);
     484        9029 :           ReplaceNode(SDValue(Node, 0), Result);
     485             :           break;
     486             :         }
     487             :         }
     488             :         return;
     489             :       }
     490             :     } else {
     491       30034 :       SDValue Value = ST->getValue();
     492             : 
     493       30034 :       EVT StVT = ST->getMemoryVT();
     494       30034 :       unsigned StWidth = StVT.getSizeInBits();
     495       60068 :       auto &DL = DAG.getDataLayout();
     496             : 
     497       30034 :       if (StWidth != StVT.getStoreSizeInBits()) {
     498             :         // Promote to a byte-sized store with upper bits zero if not
     499             :         // storing an integral number of bytes.  For example, promote
     500             :         // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
     501        3906 :         EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
     502        7812 :                                     StVT.getStoreSizeInBits());
     503        3906 :         Value = DAG.getZeroExtendInReg(Value, dl, StVT);
     504             :         SDValue Result =
     505        7812 :             DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
     506        7812 :                               Alignment, MMOFlags, AAInfo);
     507        3906 :         ReplaceNode(SDValue(Node, 0), Result);
     508       26128 :       } else if (StWidth & (StWidth - 1)) {
     509             :         // If not storing a power-of-2 number of bits, expand as two stores.
     510             :         assert(!StVT.isVector() && "Unsupported truncstore!");
     511         685 :         unsigned RoundWidth = 1 << Log2_32(StWidth);
     512             :         assert(RoundWidth < StWidth);
     513         685 :         unsigned ExtraWidth = StWidth - RoundWidth;
     514             :         assert(ExtraWidth < RoundWidth);
     515             :         assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
     516             :                "Store size not an integral number of bytes!");
     517         685 :         EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
     518         685 :         EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
     519         685 :         SDValue Lo, Hi;
     520             :         unsigned IncrementSize;
     521             : 
     522         685 :         if (DL.isLittleEndian()) {
     523             :           // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
     524             :           // Store the bottom RoundWidth bits.
     525        2025 :           Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     526         675 :                                  RoundVT, Alignment, MMOFlags, AAInfo);
     527             : 
     528             :           // Store the remaining ExtraWidth bits.
     529         675 :           IncrementSize = RoundWidth / 8;
     530        1350 :           Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     531             :                             DAG.getConstant(IncrementSize, dl,
     532        2025 :                                             Ptr.getValueType()));
     533        1350 :           Hi = DAG.getNode(
     534             :               ISD::SRL, dl, Value.getValueType(), Value,
     535             :               DAG.getConstant(RoundWidth, dl,
     536        2025 :                               TLI.getShiftAmountTy(Value.getValueType(), DL)));
     537        1350 :           Hi = DAG.getTruncStore(
     538             :               Chain, dl, Hi, Ptr,
     539         675 :               ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
     540        2700 :               MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
     541             :         } else {
     542             :           // Big endian - avoid unaligned stores.
     543             :           // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
     544             :           // Store the top RoundWidth bits.
     545          20 :           Hi = DAG.getNode(
     546             :               ISD::SRL, dl, Value.getValueType(), Value,
     547             :               DAG.getConstant(ExtraWidth, dl,
     548          30 :                               TLI.getShiftAmountTy(Value.getValueType(), DL)));
     549          30 :           Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
     550          10 :                                  RoundVT, Alignment, MMOFlags, AAInfo);
     551             : 
     552             :           // Store the remaining ExtraWidth bits.
     553          10 :           IncrementSize = RoundWidth / 8;
     554          20 :           Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     555             :                             DAG.getConstant(IncrementSize, dl,
     556          30 :                                             Ptr.getValueType()));
     557          20 :           Lo = DAG.getTruncStore(
     558             :               Chain, dl, Value, Ptr,
     559          10 :               ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
     560          40 :               MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
     561             :         }
     562             : 
     563             :         // The order of the stores doesn't matter.
     564        2055 :         SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
     565         685 :         ReplaceNode(SDValue(Node, 0), Result);
     566             :       } else {
     567      101772 :         switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
     568           0 :         default: llvm_unreachable("This action is not supported yet!");
     569       22139 :         case TargetLowering::Legal: {
     570       22139 :           EVT MemVT = ST->getMemoryVT();
     571       44278 :           unsigned AS = ST->getAddressSpace();
     572       44278 :           unsigned Align = ST->getAlignment();
     573             :           // If this is an unaligned store and the target doesn't support it,
     574             :           // expand it.
     575       22139 :           if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     576         415 :             SDValue Result = TLI.expandUnalignedStore(ST, DAG);
     577         415 :             ReplaceNode(SDValue(ST, 0), Result);
     578             :           }
     579             :           break;
     580             :         }
     581        1639 :         case TargetLowering::Custom: {
     582        3278 :           SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
     583        3138 :           if (Res && Res != SDValue(Node, 0))
     584        1499 :             ReplaceNode(SDValue(Node, 0), Res);
     585             :           return;
     586             :         }
     587        1665 :         case TargetLowering::Expand:
     588             :           assert(!StVT.isVector() &&
     589             :                  "Vector Stores are handled in LegalizeVectorOps");
     590             : 
     591             :           // TRUNCSTORE:i16 i32 -> STORE i16
     592             :           assert(TLI.isTypeLegal(StVT) &&
     593             :                  "Do not know how to expand this store!");
     594        3330 :           Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
     595             :           SDValue Result =
     596        3330 :               DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
     597        3330 :                            Alignment, MMOFlags, AAInfo);
     598        1665 :           ReplaceNode(SDValue(Node, 0), Result);
     599        1665 :           break;
     600             :         }
     601             :       }
     602             :     }
     603             : }
     604             : 
     605     1061607 : void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
     606     1061607 :   LoadSDNode *LD = cast<LoadSDNode>(Node);
     607     2123214 :   SDValue Chain = LD->getChain();  // The chain.
     608     1061607 :   SDValue Ptr = LD->getBasePtr();  // The base pointer.
     609     1061607 :   SDValue Value;                   // The value returned by the load op.
     610     1113363 :   SDLoc dl(Node);
     611             : 
     612     1061607 :   ISD::LoadExtType ExtType = LD->getExtensionType();
     613     1061607 :   if (ExtType == ISD::NON_EXTLOAD) {
     614     1009851 :     MVT VT = Node->getSimpleValueType(0);
     615     1009851 :     SDValue RVal = SDValue(Node, 0);
     616     1009851 :     SDValue RChain = SDValue(Node, 1);
     617             : 
     618     4039404 :     switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
     619           0 :     default: llvm_unreachable("This action is not supported yet!");
     620      834581 :     case TargetLowering::Legal: {
     621      834581 :       EVT MemVT = LD->getMemoryVT();
     622     1669162 :       unsigned AS = LD->getAddressSpace();
     623     1669162 :       unsigned Align = LD->getAlignment();
     624     1669162 :       const DataLayout &DL = DAG.getDataLayout();
     625             :       // If this is an unaligned load and the target doesn't support it,
     626             :       // expand it.
     627      834581 :       if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     628         684 :         std::tie(RVal, RChain) =  TLI.expandUnalignedLoad(LD, DAG);
     629             :       }
     630             :       break;
     631             :     }
     632      124567 :     case TargetLowering::Custom: {
     633      124567 :       if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
     634       15877 :         RVal = Res;
     635       31754 :         RChain = Res.getValue(1);
     636             :       }
     637      124567 :       break;
     638             :     }
     639       50703 :     case TargetLowering::Promote: {
     640       50703 :       MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
     641             :       assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
     642             :              "Can only promote loads to same size type");
     643             : 
     644      101406 :       SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
     645      152109 :       RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
     646      101406 :       RChain = Res.getValue(1);
     647             :       break;
     648             :     }
     649             :     }
     650     1009851 :     if (RChain.getNode() != Node) {
     651             :       assert(RVal.getNode() != Node && "Load must be completely replaced");
     652      128566 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
     653      128566 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
     654       64283 :       if (UpdatedNodes) {
     655        1699 :         UpdatedNodes->insert(RVal.getNode());
     656        1699 :         UpdatedNodes->insert(RChain.getNode());
     657             :       }
     658       64283 :       ReplacedNode(Node);
     659             :     }
     660             :     return;
     661             :   }
     662             : 
     663       51756 :   EVT SrcVT = LD->getMemoryVT();
     664       51756 :   unsigned SrcWidth = SrcVT.getSizeInBits();
     665      103512 :   unsigned Alignment = LD->getAlignment();
     666       51756 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
     667      103512 :   AAMDNodes AAInfo = LD->getAAInfo();
     668             : 
     669       51756 :   if (SrcWidth != SrcVT.getStoreSizeInBits() &&
     670             :       // Some targets pretend to have an i1 loading operation, and actually
     671             :       // load an i8.  This trick is correct for ZEXTLOAD because the top 7
     672             :       // bits are guaranteed to be zero; it helps the optimizers understand
     673             :       // that these bits are zero.  It is also useful for EXTLOAD, since it
     674             :       // tells the optimizers that those bits are undefined.  It would be
     675             :       // nice to have an effective generic way of getting these benefits...
     676             :       // Until such a way is found, don't insist on promoting i1 here.
     677        1309 :       (SrcVT != MVT::i1 ||
     678        1932 :        TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
     679             :          TargetLowering::Promote)) {
     680             :     // Promote to a byte-sized load if not loading an integral number of
     681             :     // bytes.  For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
     682         611 :     unsigned NewWidth = SrcVT.getStoreSizeInBits();
     683         611 :     EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
     684         611 :     SDValue Ch;
     685             : 
     686             :     // The extra bits are guaranteed to be zero, since we stored them that
     687             :     // way.  A zext load from NVT thus automatically gives zext from SrcVT.
     688             : 
     689         611 :     ISD::LoadExtType NewExtType =
     690         611 :       ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
     691             : 
     692             :     SDValue Result =
     693         611 :         DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
     694        1833 :                        LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
     695             : 
     696        1222 :     Ch = Result.getValue(1); // The chain.
     697             : 
     698         611 :     if (ExtType == ISD::SEXTLOAD)
     699             :       // Having the top bits zero doesn't help when sign extending.
     700         162 :       Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
     701             :                            Result.getValueType(),
     702         162 :                            Result, DAG.getValueType(SrcVT));
     703         798 :     else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
     704             :       // All the top bits are guaranteed to be zero - inform the optimizers.
     705         670 :       Result = DAG.getNode(ISD::AssertZext, dl,
     706             :                            Result.getValueType(), Result,
     707         670 :                            DAG.getValueType(SrcVT));
     708             : 
     709         611 :     Value = Result;
     710         611 :     Chain = Ch;
     711       51145 :   } else if (SrcWidth & (SrcWidth - 1)) {
     712             :     // If not loading a power-of-2 number of bits, expand as two loads.
     713             :     assert(!SrcVT.isVector() && "Unsupported extload!");
     714         247 :     unsigned RoundWidth = 1 << Log2_32(SrcWidth);
     715             :     assert(RoundWidth < SrcWidth);
     716         247 :     unsigned ExtraWidth = SrcWidth - RoundWidth;
     717             :     assert(ExtraWidth < RoundWidth);
     718             :     assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
     719             :            "Load size not an integral number of bytes!");
     720         247 :     EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
     721         247 :     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
     722         247 :     SDValue Lo, Hi, Ch;
     723             :     unsigned IncrementSize;
     724         494 :     auto &DL = DAG.getDataLayout();
     725             : 
     726         247 :     if (DL.isLittleEndian()) {
     727             :       // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
     728             :       // Load the bottom RoundWidth bits.
     729         444 :       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
     730         222 :                           LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
     731         666 :                           AAInfo);
     732             : 
     733             :       // Load the remaining ExtraWidth bits.
     734         222 :       IncrementSize = RoundWidth / 8;
     735         444 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     736             :                          DAG.getConstant(IncrementSize, dl,
     737         666 :                                          Ptr.getValueType()));
     738         444 :       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
     739         222 :                           LD->getPointerInfo().getWithOffset(IncrementSize),
     740         444 :                           ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
     741         888 :                           AAInfo);
     742             : 
     743             :       // Build a factor node to remember that this load is independent of
     744             :       // the other one.
     745         444 :       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
     746        1110 :                        Hi.getValue(1));
     747             : 
     748             :       // Move the top bits to the right place.
     749         444 :       Hi = DAG.getNode(
     750             :           ISD::SHL, dl, Hi.getValueType(), Hi,
     751             :           DAG.getConstant(RoundWidth, dl,
     752         666 :                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
     753             : 
     754             :       // Join the hi and lo parts.
     755         666 :       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
     756             :     } else {
     757             :       // Big endian - avoid unaligned loads.
     758             :       // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
     759             :       // Load the top RoundWidth bits.
     760          50 :       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
     761          25 :                           LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
     762          75 :                           AAInfo);
     763             : 
     764             :       // Load the remaining ExtraWidth bits.
     765          25 :       IncrementSize = RoundWidth / 8;
     766          50 :       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
     767             :                          DAG.getConstant(IncrementSize, dl,
     768          75 :                                          Ptr.getValueType()));
     769          50 :       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
     770          25 :                           LD->getPointerInfo().getWithOffset(IncrementSize),
     771          50 :                           ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
     772         100 :                           AAInfo);
     773             : 
     774             :       // Build a factor node to remember that this load is independent of
     775             :       // the other one.
     776          50 :       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
     777         125 :                        Hi.getValue(1));
     778             : 
     779             :       // Move the top bits to the right place.
     780          50 :       Hi = DAG.getNode(
     781             :           ISD::SHL, dl, Hi.getValueType(), Hi,
     782             :           DAG.getConstant(ExtraWidth, dl,
     783          75 :                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
     784             : 
     785             :       // Join the hi and lo parts.
     786          75 :       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
     787             :     }
     788             : 
     789         247 :     Chain = Ch;
     790             :   } else {
     791       50898 :     bool isCustom = false;
     792      203592 :     switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
     793             :                                  SrcVT.getSimpleVT())) {
     794           0 :     default: llvm_unreachable("This action is not supported yet!");
     795        5925 :     case TargetLowering::Custom:
     796        5925 :       isCustom = true;
     797             :       LLVM_FALLTHROUGH;
     798       50575 :     case TargetLowering::Legal: {
     799       50575 :       Value = SDValue(Node, 0);
     800       50575 :       Chain = SDValue(Node, 1);
     801             : 
     802       50575 :       if (isCustom) {
     803       11850 :         if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
     804        4345 :           Value = Res;
     805        8690 :           Chain = Res.getValue(1);
     806             :         }
     807             :       } else {
     808             :         // If this is an unaligned load and the target doesn't support it,
     809             :         // expand it.
     810       44650 :         EVT MemVT = LD->getMemoryVT();
     811       89300 :         unsigned AS = LD->getAddressSpace();
     812       89300 :         unsigned Align = LD->getAlignment();
     813       89300 :         const DataLayout &DL = DAG.getDataLayout();
     814       44650 :         if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
     815        1272 :           std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
     816             :         }
     817             :       }
     818             :       break;
     819             :     }
     820         323 :     case TargetLowering::Expand:
     821         323 :       EVT DestVT = Node->getValueType(0);
     822         646 :       if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
     823             :         // If the source type is not legal, see if there is a legal extload to
     824             :         // an intermediate type that we can then extend further.
     825         924 :         EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
     826         364 :         if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
     827         112 :             TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
     828             :           // If we are loading a legal type, this is a non-extload followed by a
     829             :           // full extend.
     830             :           ISD::LoadExtType MidExtType =
     831         308 :               (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
     832             : 
     833         308 :           SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
     834         308 :                                         SrcVT, LD->getMemOperand());
     835             :           unsigned ExtendOp =
     836         308 :               ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
     837         924 :           Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
     838         616 :           Chain = Load.getValue(1);
     839             :           break;
     840             :         }
     841             : 
     842             :         // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
     843             :         // normal undefined upper bits behavior to allow using an in-reg extend
     844             :         // with the illegal FP type, so load as an integer and do the
     845             :         // from-integer conversion.
     846           0 :         if (SrcVT.getScalarType() == MVT::f16) {
     847           0 :           EVT ISrcVT = SrcVT.changeTypeToInteger();
     848           0 :           EVT IDestVT = DestVT.changeTypeToInteger();
     849           0 :           EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
     850             : 
     851           0 :           SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
     852             :                                           Chain, Ptr, ISrcVT,
     853           0 :                                           LD->getMemOperand());
     854           0 :           Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
     855           0 :           Chain = Result.getValue(1);
     856             :           break;
     857             :         }
     858             :       }
     859             : 
     860             :       assert(!SrcVT.isVector() &&
     861             :              "Vector Loads are handled in LegalizeVectorOps");
     862             : 
     863             :       // FIXME: This does not work for vectors on most targets.  Sign-
     864             :       // and zero-extend operations are currently folded into extending
     865             :       // loads, whether they are legal or not, and then we end up here
     866             :       // without any support for legalizing them.
     867             :       assert(ExtType != ISD::EXTLOAD &&
     868             :              "EXTLOAD should always be supported!");
     869             :       // Turn the unsupported load into an EXTLOAD followed by an
     870             :       // explicit zero/sign extend inreg.
     871          15 :       SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
     872             :                                       Node->getValueType(0),
     873             :                                       Chain, Ptr, SrcVT,
     874          30 :                                       LD->getMemOperand());
     875          15 :       SDValue ValRes;
     876          15 :       if (ExtType == ISD::SEXTLOAD)
     877          28 :         ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
     878             :                              Result.getValueType(),
     879          28 :                              Result, DAG.getValueType(SrcVT));
     880             :       else
     881           1 :         ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
     882          15 :       Value = ValRes;
     883          30 :       Chain = Result.getValue(1);
     884          15 :       break;
     885             :     }
     886             :   }
     887             : 
     888             :   // Since loads produce two values, make sure to remember that we legalized
     889             :   // both of them.
     890       51756 :   if (Chain.getNode() != Node) {
     891             :     assert(Value.getNode() != Node && "Load must be completely replaced");
     892       11872 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
     893       11872 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
     894        5936 :     if (UpdatedNodes) {
     895           1 :       UpdatedNodes->insert(Value.getNode());
     896           1 :       UpdatedNodes->insert(Chain.getNode());
     897             :     }
     898        5936 :     ReplacedNode(Node);
     899             :   }
     900             : }
     901             : 
     902             : static TargetLowering::LegalizeAction
     903          34 : getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) {
     904             :   unsigned EqOpc;
     905          34 :   switch (Opcode) {
     906           0 :     default: llvm_unreachable("Unexpected FP pseudo-opcode");
     907             :     case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
     908           2 :     case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
     909           2 :     case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
     910           6 :     case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
     911           2 :     case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
     912           2 :     case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
     913           2 :     case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
     914           2 :     case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
     915           2 :     case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
     916           2 :     case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
     917           2 :     case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
     918           3 :     case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
     919           3 :     case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
     920             :   }
     921             : 
     922          68 :   auto Action = TLI.getOperationAction(EqOpc, VT);
     923             : 
     924             :   // We don't currently handle Custom or Promote for strict FP pseudo-ops.
     925             :   // For now, we just expand for those cases.
     926          34 :   if (Action != TargetLowering::Legal)
     927          22 :     Action = TargetLowering::Expand;
     928             : 
     929          34 :   return Action;
     930             : }
     931             : 
     932             : /// Return a legal replacement for the given operation, with all legal operands.
     933    20832165 : void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     934             :   DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
     935             : 
     936    41664330 :   if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
     937             :     return;
     938             : 
     939             : #ifndef NDEBUG
     940             :   for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
     941             :     assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
     942             :               TargetLowering::TypeLegal ||
     943             :             TLI.isTypeLegal(Node->getValueType(i))) &&
     944             :            "Unexpected illegal type!");
     945             : 
     946             :   for (const SDValue &Op : Node->op_values())
     947             :     assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
     948             :               TargetLowering::TypeLegal ||
     949             :             TLI.isTypeLegal(Op.getValueType()) ||
     950             :             Op.getOpcode() == ISD::TargetConstant) &&
     951             :             "Unexpected illegal type!");
     952             : #endif
     953             : 
     954             :   // Figure out the correct action; the way to query this varies by opcode
     955    20381930 :   TargetLowering::LegalizeAction Action = TargetLowering::Legal;
     956    20381930 :   bool SimpleFinishLegalizing = true;
     957    20381930 :   switch (Node->getOpcode()) {
     958       50650 :   case ISD::INTRINSIC_W_CHAIN:
     959             :   case ISD::INTRINSIC_WO_CHAIN:
     960             :   case ISD::INTRINSIC_VOID:
     961             :   case ISD::STACKSAVE:
     962      151950 :     Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
     963             :     break;
     964           4 :   case ISD::GET_DYNAMIC_AREA_OFFSET:
     965           8 :     Action = TLI.getOperationAction(Node->getOpcode(),
     966             :                                     Node->getValueType(0));
     967             :     break;
     968         192 :   case ISD::VAARG:
     969         576 :     Action = TLI.getOperationAction(Node->getOpcode(),
     970             :                                     Node->getValueType(0));
     971         192 :     if (Action != TargetLowering::Promote)
     972         573 :       Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
     973             :     break;
     974      217328 :   case ISD::FP_TO_FP16:
     975             :   case ISD::SINT_TO_FP:
     976             :   case ISD::UINT_TO_FP:
     977             :   case ISD::EXTRACT_VECTOR_ELT:
     978      434656 :     Action = TLI.getOperationAction(Node->getOpcode(),
     979      434656 :                                     Node->getOperand(0).getValueType());
     980             :     break;
     981       24813 :   case ISD::FP_ROUND_INREG:
     982             :   case ISD::SIGN_EXTEND_INREG: {
     983       74439 :     EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
     984       49626 :     Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
     985             :     break;
     986             :   }
     987         986 :   case ISD::ATOMIC_STORE: {
     988        1972 :     Action = TLI.getOperationAction(Node->getOpcode(),
     989        1972 :                                     Node->getOperand(2).getValueType());
     990             :     break;
     991             :   }
     992       73642 :   case ISD::SELECT_CC:
     993             :   case ISD::SETCC:
     994             :   case ISD::BR_CC: {
     995      129372 :     unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
     996       50856 :                          Node->getOpcode() == ISD::SETCC ? 2 :
     997       78516 :                          Node->getOpcode() == ISD::SETCCE ? 3 : 1;
     998       73642 :     unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
     999      220926 :     MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
    1000             :     ISD::CondCode CCCode =
    1001      220926 :         cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
    1002      147284 :     Action = TLI.getCondCodeAction(CCCode, OpVT);
    1003       73642 :     if (Action == TargetLowering::Legal) {
    1004       73007 :       if (Node->getOpcode() == ISD::SELECT_CC)
    1005       44970 :         Action = TLI.getOperationAction(Node->getOpcode(),
    1006             :                                         Node->getValueType(0));
    1007             :       else
    1008      151566 :         Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
    1009             :     }
    1010             :     break;
    1011             :   }
    1012             :   case ISD::LOAD:
    1013             :   case ISD::STORE:
    1014             :     // FIXME: Model these properly.  LOAD and STORE are complicated, and
    1015             :     // STORE expects the unlegalized operand in some cases.
    1016             :     SimpleFinishLegalizing = false;
    1017             :     break;
    1018             :   case ISD::CALLSEQ_START:
    1019             :   case ISD::CALLSEQ_END:
    1020             :     // FIXME: This shouldn't be necessary.  These nodes have special properties
    1021             :     // dealing with the recursive nature of legalization.  Removing this
    1022             :     // special case should be done as part of making LegalizeDAG non-recursive.
    1023             :     SimpleFinishLegalizing = false;
    1024             :     break;
    1025      143814 :   case ISD::EXTRACT_ELEMENT:
    1026             :   case ISD::FLT_ROUNDS_:
    1027             :   case ISD::MERGE_VALUES:
    1028             :   case ISD::EH_RETURN:
    1029             :   case ISD::FRAME_TO_ARGS_OFFSET:
    1030             :   case ISD::EH_DWARF_CFA:
    1031             :   case ISD::EH_SJLJ_SETJMP:
    1032             :   case ISD::EH_SJLJ_LONGJMP:
    1033             :   case ISD::EH_SJLJ_SETUP_DISPATCH:
    1034             :     // These operations lie about being legal: when they claim to be legal,
    1035             :     // they should actually be expanded.
    1036      431442 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1037      143814 :     if (Action == TargetLowering::Legal)
    1038             :       Action = TargetLowering::Expand;
    1039             :     break;
    1040         191 :   case ISD::INIT_TRAMPOLINE:
    1041             :   case ISD::ADJUST_TRAMPOLINE:
    1042             :   case ISD::FRAMEADDR:
    1043             :   case ISD::RETURNADDR:
    1044             :   case ISD::ADDROFRETURNADDR:
    1045             :     // These operations lie about being legal: when they claim to be legal,
    1046             :     // they should actually be custom-lowered.
    1047         573 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1048         191 :     if (Action == TargetLowering::Legal)
    1049             :       Action = TargetLowering::Custom;
    1050             :     break;
    1051          25 :   case ISD::READCYCLECOUNTER:
    1052             :     // READCYCLECOUNTER returns an i64, even if type legalization might have
    1053             :     // expanded that to several smaller types.
    1054          75 :     Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
    1055             :     break;
    1056             :   case ISD::READ_REGISTER:
    1057             :   case ISD::WRITE_REGISTER:
    1058             :     // Named register is legal in the DAG, but blocked by register name
    1059             :     // selection if not implemented by target (to chose the correct register)
    1060             :     // They'll be converted to Copy(To/From)Reg.
    1061             :     Action = TargetLowering::Legal;
    1062             :     break;
    1063          26 :   case ISD::DEBUGTRAP:
    1064          78 :     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1065          26 :     if (Action == TargetLowering::Expand) {
    1066             :       // replace ISD::DEBUGTRAP with ISD::TRAP
    1067          11 :       SDValue NewVal;
    1068          33 :       NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
    1069          33 :                            Node->getOperand(0));
    1070          11 :       ReplaceNode(Node, NewVal.getNode());
    1071          11 :       LegalizeOp(NewVal.getNode());
    1072             :       return;
    1073          15 :     }
    1074             :     break;
    1075          34 :   case ISD::STRICT_FSQRT:
    1076             :   case ISD::STRICT_FMA:
    1077             :   case ISD::STRICT_FPOW:
    1078             :   case ISD::STRICT_FPOWI:
    1079             :   case ISD::STRICT_FSIN:
    1080             :   case ISD::STRICT_FCOS:
    1081             :   case ISD::STRICT_FEXP:
    1082             :   case ISD::STRICT_FEXP2:
    1083             :   case ISD::STRICT_FLOG:
    1084             :   case ISD::STRICT_FLOG10:
    1085             :   case ISD::STRICT_FLOG2:
    1086             :   case ISD::STRICT_FRINT:
    1087             :   case ISD::STRICT_FNEARBYINT:
    1088             :     // These pseudo-ops get legalized as if they were their non-strict
    1089             :     // equivalent.  For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
    1090             :     // is also legal, but if ISD::FSQRT requires expansion then so does
    1091             :     // ISD::STRICT_FSQRT.
    1092          68 :     Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(),
    1093             :                                      Node->getValueType(0));
    1094             :     break;
    1095             : 
    1096    16567927 :   default:
    1097    16567927 :     if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
    1098             :       Action = TargetLowering::Legal;
    1099             :     } else {
    1100    27556264 :       Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
    1101             :     }
    1102             :     break;
    1103             :   }
    1104             : 
    1105             :   if (SimpleFinishLegalizing) {
    1106    17080373 :     SDNode *NewNode = Node;
    1107    34160746 :     switch (Node->getOpcode()) {
    1108             :     default: break;
    1109      260071 :     case ISD::SHL:
    1110             :     case ISD::SRL:
    1111             :     case ISD::SRA:
    1112             :     case ISD::ROTL:
    1113             :     case ISD::ROTR: {
    1114             :       // Legalizing shifts/rotates requires adjusting the shift amount
    1115             :       // to the appropriate width.
    1116      520142 :       SDValue Op0 = Node->getOperand(0);
    1117      520142 :       SDValue Op1 = Node->getOperand(1);
    1118      780213 :       if (!Op1.getValueType().isVector()) {
    1119      511234 :         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
    1120             :         // The getShiftAmountOperand() may create a new operand node or
    1121             :         // return the existing one. If new operand is created we need
    1122             :         // to update the parent node.
    1123             :         // Do not try to legalize SAO here! It will be automatically legalized
    1124             :         // in the next round.
    1125       37680 :         if (SAO != Op1)
    1126       37680 :           NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
    1127      260071 :       }
    1128             :     }
    1129      260071 :     break;
    1130         428 :     case ISD::SRL_PARTS:
    1131             :     case ISD::SRA_PARTS:
    1132             :     case ISD::SHL_PARTS: {
    1133             :       // Legalizing shifts/rotates requires adjusting the shift amount
    1134             :       // to the appropriate width.
    1135         856 :       SDValue Op0 = Node->getOperand(0);
    1136         856 :       SDValue Op1 = Node->getOperand(1);
    1137         856 :       SDValue Op2 = Node->getOperand(2);
    1138        1284 :       if (!Op2.getValueType().isVector()) {
    1139         856 :         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
    1140             :         // The getShiftAmountOperand() may create a new operand node or
    1141             :         // return the existing one. If new operand is created we need
    1142             :         // to update the parent node.
    1143           0 :         if (SAO != Op2)
    1144           0 :           NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
    1145         428 :       }
    1146             :     }
    1147         428 :     break;
    1148             :     }
    1149             : 
    1150    17080373 :     if (NewNode != Node) {
    1151         106 :       ReplaceNode(Node, NewNode);
    1152         106 :       Node = NewNode;
    1153             :     }
    1154    17080373 :     switch (Action) {
    1155             :     case TargetLowering::Legal:
    1156             :       return;
    1157      988102 :     case TargetLowering::Custom: {
    1158             :       // FIXME: The handling for custom lowering with multiple results is
    1159             :       // a complete mess.
    1160     1976204 :       if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
    1161      948692 :         if (!(Res.getNode() != Node || Res.getResNo() != 0))
    1162             :           return;
    1163             : 
    1164      767993 :         if (Node->getNumValues() == 1) {
    1165             :           // We can just directly replace this node with the lowered value.
    1166      634444 :           ReplaceNode(SDValue(Node, 0), Res);
    1167      634444 :           return;
    1168             :         }
    1169             : 
    1170      133549 :         SmallVector<SDValue, 8> ResultVals;
    1171      534218 :         for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
    1172      534240 :           ResultVals.push_back(Res.getValue(i));
    1173      267098 :         ReplaceNode(Node, ResultVals.data());
    1174             :         return;
    1175             :       }
    1176             :       LLVM_FALLTHROUGH;
    1177             :     }
    1178             :     case TargetLowering::Expand:
    1179      228092 :       if (ExpandNode(Node))
    1180             :         return;
    1181             :       LLVM_FALLTHROUGH;
    1182             :     case TargetLowering::LibCall:
    1183       49292 :       ConvertNodeToLibcall(Node);
    1184       49292 :       return;
    1185        6124 :     case TargetLowering::Promote:
    1186        6124 :       PromoteNode(Node);
    1187        6124 :       return;
    1188             :     }
    1189             :   }
    1190             : 
    1191     6603092 :   switch (Node->getOpcode()) {
    1192           0 :   default:
    1193             : #ifndef NDEBUG
    1194             :     dbgs() << "NODE: ";
    1195             :     Node->dump( &DAG);
    1196             :     dbgs() << "\n";
    1197             : #endif
    1198           0 :     llvm_unreachable("Do not know how to legalize this operator!");
    1199             : 
    1200             :   case ISD::CALLSEQ_START:
    1201             :   case ISD::CALLSEQ_END:
    1202             :     break;
    1203     1061607 :   case ISD::LOAD: {
    1204     1061607 :     return LegalizeLoadOps(Node);
    1205             :   }
    1206     1466538 :   case ISD::STORE: {
    1207     1466538 :     return LegalizeStoreOps(Node);
    1208             :   }
    1209             :   }
    1210             : }
    1211             : 
    1212        2061 : SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
    1213        4122 :   SDValue Vec = Op.getOperand(0);
    1214        4122 :   SDValue Idx = Op.getOperand(1);
    1215        4122 :   SDLoc dl(Op);
    1216             : 
    1217             :   // Before we generate a new store to a temporary stack slot, see if there is
    1218             :   // already one that we can use. There often is because when we scalarize
    1219             :   // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
    1220             :   // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
    1221             :   // the vector. If all are expanded here, we don't want one store per vector
    1222             :   // element.
    1223             : 
    1224             :   // Caches for hasPredecessorHelper
    1225        4122 :   SmallPtrSet<const SDNode *, 32> Visited;
    1226        4122 :   SmallVector<const SDNode *, 16> Worklist;
    1227        2061 :   Worklist.push_back(Idx.getNode());
    1228        2061 :   SDValue StackPtr, Ch;
    1229        2061 :   for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
    1230       10331 :        UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
    1231        3730 :     SDNode *User = *UI;
    1232        1645 :     if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
    1233        4935 :       if (ST->isIndexed() || ST->isTruncatingStore() ||
    1234        3290 :           ST->getValue() != Vec)
    1235             :         continue;
    1236             : 
    1237             :       // Make sure that nothing else could have stored into the destination of
    1238             :       // this store.
    1239        4935 :       if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
    1240             :         continue;
    1241             : 
    1242             :       // If the index is dependent on the store we will introduce a cycle when
    1243             :       // creating the load (the load uses the index, and by replacing the chain
    1244             :       // we will make the index dependent on the load). Also, the store might be
    1245             :       // dependent on the extractelement and introduce a cycle when creating
    1246             :       // the load.
    1247        3291 :       if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
    1248        1644 :           ST->hasPredecessor(Op.getNode()))
    1249             :         continue;
    1250             : 
    1251        1643 :       StackPtr = ST->getBasePtr();
    1252        1643 :       Ch = SDValue(ST, 0);
    1253             :       break;
    1254             :     }
    1255             :   }
    1256             : 
    1257        4122 :   EVT VecVT = Vec.getValueType();
    1258             : 
    1259        2061 :   if (!Ch.getNode()) {
    1260             :     // Store the value to a temporary stack slot, then LOAD the returned part.
    1261         418 :     StackPtr = DAG.CreateStackTemporary(VecVT);
    1262        1254 :     Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
    1263        1672 :                       MachinePointerInfo());
    1264             :   }
    1265             : 
    1266        2061 :   StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1267             : 
    1268        2061 :   SDValue NewLoad;
    1269             : 
    1270        6183 :   if (Op.getValueType().isVector())
    1271           0 :     NewLoad =
    1272           0 :         DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
    1273             :   else
    1274        4122 :     NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
    1275             :                              MachinePointerInfo(),
    1276        6183 :                              VecVT.getVectorElementType());
    1277             : 
    1278             :   // Replace the chain going out of the store, by the one out of the load.
    1279        4122 :   DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
    1280             : 
    1281             :   // We introduced a cycle though, so update the loads operands, making sure
    1282             :   // to use the original store's chain as an incoming chain.
    1283             :   SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
    1284        8244 :                                           NewLoad->op_end());
    1285        2061 :   NewLoadOperands[0] = Ch;
    1286        2061 :   NewLoad =
    1287        6183 :       SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
    1288        4122 :   return NewLoad;
    1289             : }
    1290             : 
    1291           0 : SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
    1292             :   assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
    1293             : 
    1294           0 :   SDValue Vec  = Op.getOperand(0);
    1295           0 :   SDValue Part = Op.getOperand(1);
    1296           0 :   SDValue Idx  = Op.getOperand(2);
    1297           0 :   SDLoc dl(Op);
    1298             : 
    1299             :   // Store the value to a temporary stack slot, then LOAD the returned part.
    1300           0 :   EVT VecVT = Vec.getValueType();
    1301           0 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1302           0 :   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1303             :   MachinePointerInfo PtrInfo =
    1304           0 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
    1305             : 
    1306             :   // First store the whole vector.
    1307           0 :   SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
    1308             : 
    1309             :   // Then store the inserted part.
    1310           0 :   SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1311             : 
    1312             :   // Store the subvector.
    1313           0 :   Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
    1314             : 
    1315             :   // Finally, load the updated vector.
    1316           0 :   return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
    1317             : }
    1318             : 
    1319         144 : SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
    1320             :   // We can't handle this case efficiently.  Allocate a sufficiently
    1321             :   // aligned object on the stack, store each element into it, then load
    1322             :   // the result as a vector.
    1323             :   // Create the stack frame object.
    1324         288 :   EVT VT = Node->getValueType(0);
    1325         144 :   EVT EltVT = VT.getVectorElementType();
    1326         288 :   SDLoc dl(Node);
    1327         144 :   SDValue FIPtr = DAG.CreateStackTemporary(VT);
    1328         288 :   int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
    1329             :   MachinePointerInfo PtrInfo =
    1330         144 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
    1331             : 
    1332             :   // Emit a store of each element to the stack slot.
    1333         288 :   SmallVector<SDValue, 8> Stores;
    1334         144 :   unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
    1335             :   // Store (in the right endianness) the elements to memory.
    1336        1304 :   for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
    1337             :     // Ignore undef elements.
    1338        3048 :     if (Node->getOperand(i).isUndef()) continue;
    1339             : 
    1340         891 :     unsigned Offset = TypeByteSize*i;
    1341             : 
    1342        1782 :     SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
    1343        2673 :     Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
    1344             : 
    1345             :     // If the destination vector element type is narrower than the source
    1346             :     // element type, only store the bits necessary.
    1347        2673 :     if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
    1348        1818 :       Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
    1349        1212 :                                          Node->getOperand(i), Idx,
    1350        2424 :                                          PtrInfo.getWithOffset(Offset), EltVT));
    1351             :     } else
    1352        1140 :       Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
    1353        1140 :                                     Idx, PtrInfo.getWithOffset(Offset)));
    1354             :   }
    1355             : 
    1356         144 :   SDValue StoreChain;
    1357         144 :   if (!Stores.empty())    // Not all undef elements?
    1358         576 :     StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
    1359             :   else
    1360           0 :     StoreChain = DAG.getEntryNode();
    1361             : 
    1362             :   // Result is a load from the stack slot.
    1363         432 :   return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
    1364             : }
    1365             : 
    1366             : namespace {
    1367             : /// Keeps track of state when getting the sign of a floating-point value as an
    1368             : /// integer.
    1369         827 : struct FloatSignAsInt {
    1370             :   EVT FloatVT;
    1371             :   SDValue Chain;
    1372             :   SDValue FloatPtr;
    1373             :   SDValue IntPtr;
    1374             :   MachinePointerInfo IntPointerInfo;
    1375             :   MachinePointerInfo FloatPointerInfo;
    1376             :   SDValue IntValue;
    1377             :   APInt SignMask;
    1378             :   uint8_t SignBit;
    1379             : };
    1380             : }
    1381             : 
    1382             : /// Bitcast a floating-point value to an integer value. Only bitcast the part
    1383             : /// containing the sign bit if the target has no integer value capable of
    1384             : /// holding all bits of the floating-point value.
    1385          79 : void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
    1386             :                                              const SDLoc &DL,
    1387             :                                              SDValue Value) const {
    1388         158 :   EVT FloatVT = Value.getValueType();
    1389          79 :   unsigned NumBits = FloatVT.getSizeInBits();
    1390          79 :   State.FloatVT = FloatVT;
    1391          79 :   EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
    1392             :   // Convert to an integer of the same size.
    1393         154 :   if (TLI.isTypeLegal(IVT)) {
    1394         150 :     State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
    1395         225 :     State.SignMask = APInt::getSignMask(NumBits);
    1396          75 :     State.SignBit = NumBits - 1;
    1397          75 :     return;
    1398             :   }
    1399             : 
    1400           8 :   auto &DataLayout = DAG.getDataLayout();
    1401             :   // Store the float to memory, then load the sign part out as an integer.
    1402           8 :   MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
    1403             :   // First create a temporary that is aligned for both the load and store.
    1404           8 :   SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
    1405           8 :   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1406             :   // Then store the float to it.
    1407           4 :   State.FloatPtr = StackPtr;
    1408           4 :   MachineFunction &MF = DAG.getMachineFunction();
    1409           4 :   State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
    1410          12 :   State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
    1411          12 :                              State.FloatPointerInfo);
    1412             : 
    1413           4 :   SDValue IntPtr;
    1414           4 :   if (DataLayout.isBigEndian()) {
    1415             :     assert(FloatVT.isByteSized() && "Unsupported floating point type!");
    1416             :     // Load out a legal integer with the same sign bit as the float.
    1417           1 :     IntPtr = StackPtr;
    1418           1 :     State.IntPointerInfo = State.FloatPointerInfo;
    1419             :   } else {
    1420             :     // Advance the pointer so that the loaded byte will contain the sign bit.
    1421           3 :     unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
    1422           6 :     IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
    1423           9 :                       DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
    1424           3 :     State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
    1425           3 :                                                              ByteOffset);
    1426             :   }
    1427             : 
    1428           4 :   State.IntPtr = IntPtr;
    1429           8 :   State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
    1430          16 :                                   State.IntPointerInfo, MVT::i8);
    1431          12 :   State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
    1432           4 :   State.SignBit = 7;
    1433             : }
    1434             : 
    1435             : /// Replace the integer value produced by getSignAsIntValue() with a new value
    1436             : /// and cast the result back to a floating-point type.
    1437          37 : SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
    1438             :                                               const SDLoc &DL,
    1439             :                                               SDValue NewIntValue) const {
    1440          37 :   if (!State.Chain)
    1441          72 :     return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
    1442             : 
    1443             :   // Override the part containing the sign bit in the value stored on the stack.
    1444           1 :   SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
    1445           2 :                                     State.IntPointerInfo, MVT::i8);
    1446           1 :   return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
    1447           1 :                      State.FloatPointerInfo);
    1448             : }
    1449             : 
    1450          42 : SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
    1451          84 :   SDLoc DL(Node);
    1452          84 :   SDValue Mag = Node->getOperand(0);
    1453          84 :   SDValue Sign = Node->getOperand(1);
    1454             : 
    1455             :   // Get sign bit into an integer value.
    1456          84 :   FloatSignAsInt SignAsInt;
    1457          42 :   getSignAsIntValue(SignAsInt, DL, Sign);
    1458             : 
    1459          84 :   EVT IntVT = SignAsInt.IntValue.getValueType();
    1460          42 :   SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
    1461          42 :   SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
    1462          42 :                                 SignMask);
    1463             : 
    1464             :   // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
    1465          84 :   EVT FloatVT = Mag.getValueType();
    1466          42 :   if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
    1467           5 :       TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
    1468          10 :     SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
    1469          10 :     SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
    1470           5 :     SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
    1471           5 :                                 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
    1472           5 :     return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
    1473             :   }
    1474             : 
    1475             :   // Transform Mag value to integer, and clear the sign bit.
    1476          37 :   FloatSignAsInt MagAsInt;
    1477          37 :   getSignAsIntValue(MagAsInt, DL, Mag);
    1478          74 :   EVT MagVT = MagAsInt.IntValue.getValueType();
    1479         185 :   SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
    1480          37 :   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
    1481          37 :                                     ClearSignMask);
    1482             : 
    1483             :   // Get the signbit at the right position for MagAsInt.
    1484          37 :   int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
    1485          37 :   if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
    1486          19 :     if (ShiftAmount > 0) {
    1487          19 :       SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
    1488          38 :       SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
    1489           0 :     } else if (ShiftAmount < 0) {
    1490           0 :       SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
    1491           0 :       SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
    1492             :     }
    1493          38 :     SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
    1494          18 :   } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
    1495           0 :     SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
    1496           0 :     if (ShiftAmount > 0) {
    1497           0 :       SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
    1498           0 :       SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
    1499           0 :     } else if (ShiftAmount < 0) {
    1500           0 :       SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
    1501           0 :       SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
    1502             :     }
    1503             :   }
    1504             : 
    1505             :   // Store the part with the modified sign and convert back to float.
    1506          74 :   SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
    1507          37 :   return modifySignAsInt(MagAsInt, DL, CopiedSign);
    1508             : }
    1509             : 
    1510           2 : SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
    1511           4 :   SDLoc DL(Node);
    1512           4 :   SDValue Value = Node->getOperand(0);
    1513             : 
    1514             :   // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
    1515           4 :   EVT FloatVT = Value.getValueType();
    1516           2 :   if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
    1517           2 :     SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
    1518           4 :     return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
    1519             :   }
    1520             : 
    1521             :   // Transform value to integer, clear the sign bit and transform back.
    1522           0 :   FloatSignAsInt ValueAsInt;
    1523           0 :   getSignAsIntValue(ValueAsInt, DL, Value);
    1524           0 :   EVT IntVT = ValueAsInt.IntValue.getValueType();
    1525           0 :   SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
    1526           0 :   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
    1527           0 :                                     ClearSignMask);
    1528           0 :   return modifySignAsInt(ValueAsInt, DL, ClearedSign);
    1529             : }
    1530             : 
    1531         140 : void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
    1532             :                                            SmallVectorImpl<SDValue> &Results) {
    1533         140 :   unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
    1534             :   assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
    1535             :           " not tell us which reg is the stack pointer!");
    1536         280 :   SDLoc dl(Node);
    1537         280 :   EVT VT = Node->getValueType(0);
    1538         140 :   SDValue Tmp1 = SDValue(Node, 0);
    1539         140 :   SDValue Tmp2 = SDValue(Node, 1);
    1540         280 :   SDValue Tmp3 = Node->getOperand(2);
    1541         280 :   SDValue Chain = Tmp1.getOperand(0);
    1542             : 
    1543             :   // Chain the dynamic stack allocation so that it doesn't modify the stack
    1544             :   // pointer when other instructions are using the stack.
    1545         140 :   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
    1546             : 
    1547         280 :   SDValue Size  = Tmp2.getOperand(1);
    1548         140 :   SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
    1549         280 :   Chain = SP.getValue(1);
    1550         280 :   unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
    1551             :   unsigned StackAlign =
    1552         280 :       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
    1553         280 :   Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size);       // Value
    1554         140 :   if (Align > StackAlign)
    1555         102 :     Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
    1556         102 :                        DAG.getConstant(-(uint64_t)Align, dl, VT));
    1557         140 :   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
    1558             : 
    1559         420 :   Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
    1560         420 :                             DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
    1561             : 
    1562         140 :   Results.push_back(Tmp1);
    1563         140 :   Results.push_back(Tmp2);
    1564         140 : }
    1565             : 
    1566             : /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
    1567             : /// target.
    1568             : ///
    1569             : /// If the SETCC has been legalized using AND / OR, then the legalized node
    1570             : /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
    1571             : /// will be set to false.
    1572             : ///
    1573             : /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
    1574             : /// then the values of LHS and RHS will be swapped, CC will be set to the
    1575             : /// new condition, and NeedInvert will be set to false.
    1576             : ///
    1577             : /// If the SETCC has been legalized using the inverse condcode, then LHS and
    1578             : /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
    1579             : /// will be set to true. The caller must invert the result of the SETCC with
    1580             : /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
    1581             : /// of a true/false result.
    1582             : ///
    1583             : /// \returns true if the SetCC has been legalized, false if it hasn't.
    1584        2213 : bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
    1585             :                                                  SDValue &RHS, SDValue &CC,
    1586             :                                                  bool &NeedInvert,
    1587             :                                                  const SDLoc &dl) {
    1588        2213 :   MVT OpVT = LHS.getSimpleValueType();
    1589        2213 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
    1590        2213 :   NeedInvert = false;
    1591        4426 :   switch (TLI.getCondCodeAction(CCCode, OpVT)) {
    1592           0 :   default: llvm_unreachable("Unknown condition code action!");
    1593             :   case TargetLowering::Legal:
    1594             :     // Nothing to do.
    1595             :     break;
    1596         367 :   case TargetLowering::Expand: {
    1597         367 :     ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
    1598         501 :     if (TLI.isCondCodeLegal(InvCC, OpVT)) {
    1599         233 :       std::swap(LHS, RHS);
    1600         233 :       CC = DAG.getCondCode(InvCC);
    1601             :       return true;
    1602             :     }
    1603         134 :     ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
    1604         134 :     unsigned Opc = 0;
    1605         134 :     switch (CCCode) {
    1606           0 :     default: llvm_unreachable("Don't know how to expand this condition!");
    1607             :     case ISD::SETO:
    1608             :         assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
    1609             :             == TargetLowering::Legal
    1610             :             && "If SETO is expanded, SETOEQ must be legal!");
    1611             :         CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
    1612          15 :     case ISD::SETUO:
    1613             :         assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
    1614             :             == TargetLowering::Legal
    1615             :             && "If SETUO is expanded, SETUNE must be legal!");
    1616          15 :         CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;  break;
    1617          97 :     case ISD::SETOEQ:
    1618             :     case ISD::SETOGT:
    1619             :     case ISD::SETOGE:
    1620             :     case ISD::SETOLT:
    1621             :     case ISD::SETOLE:
    1622             :     case ISD::SETONE:
    1623             :     case ISD::SETUEQ:
    1624             :     case ISD::SETUNE:
    1625             :     case ISD::SETUGT:
    1626             :     case ISD::SETUGE:
    1627             :     case ISD::SETULT:
    1628             :     case ISD::SETULE:
    1629             :         // If we are floating point, assign and break, otherwise fall through.
    1630          97 :         if (!OpVT.isInteger()) {
    1631             :           // We can use the 4th bit to tell if we are the unordered
    1632             :           // or ordered version of the opcode.
    1633          97 :           CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
    1634          97 :           Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
    1635          97 :           CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
    1636             :           break;
    1637             :         }
    1638             :         // Fallthrough if we are unsigned integer.
    1639             :         LLVM_FALLTHROUGH;
    1640             :     case ISD::SETLE:
    1641             :     case ISD::SETGT:
    1642             :     case ISD::SETGE:
    1643             :     case ISD::SETLT:
    1644             :       // We only support using the inverted operation, which is computed above
    1645             :       // and not a different manner of supporting expanding these cases.
    1646           0 :       llvm_unreachable("Don't know how to expand this condition!");
    1647          16 :     case ISD::SETNE:
    1648             :     case ISD::SETEQ:
    1649             :       // Try inverting the result of the inverse condition.
    1650          16 :       InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
    1651          16 :       if (TLI.isCondCodeLegal(InvCC, OpVT)) {
    1652          16 :         CC = DAG.getCondCode(InvCC);
    1653          16 :         NeedInvert = true;
    1654             :         return true;
    1655             :       }
    1656             :       // If inverting the condition didn't work then we have no means to expand
    1657             :       // the condition.
    1658           0 :       llvm_unreachable("Don't know how to expand this condition!");
    1659             :     }
    1660             : 
    1661         118 :     SDValue SetCC1, SetCC2;
    1662         118 :     if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
    1663             :       // If we aren't the ordered or unorder operation,
    1664             :       // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
    1665          97 :       SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
    1666          97 :       SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
    1667             :     } else {
    1668             :       // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
    1669          21 :       SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
    1670          21 :       SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
    1671             :     }
    1672         236 :     LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
    1673         118 :     RHS = SDValue();
    1674         118 :     CC  = SDValue();
    1675             :     return true;
    1676             :   }
    1677             :   }
    1678             :   return false;
    1679             : }
    1680             : 
    1681             : /// Emit a store/load combination to the stack.  This stores
    1682             : /// SrcOp to a stack slot of type SlotVT, truncating it if needed.  It then does
    1683             : /// a load from the stack slot to DestVT, extending it if needed.
    1684             : /// The resultant code need not be legal.
    1685         428 : SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
    1686             :                                                EVT DestVT, const SDLoc &dl) {
    1687             :   // Create the stack frame object.
    1688         856 :   unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
    1689        1284 :       SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
    1690         428 :   SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
    1691             : 
    1692         428 :   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
    1693         428 :   int SPFI = StackPtrFI->getIndex();
    1694             :   MachinePointerInfo PtrInfo =
    1695         428 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
    1696             : 
    1697         428 :   unsigned SrcSize = SrcOp.getValueSizeInBits();
    1698         428 :   unsigned SlotSize = SlotVT.getSizeInBits();
    1699         428 :   unsigned DestSize = DestVT.getSizeInBits();
    1700         428 :   Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
    1701         856 :   unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
    1702             : 
    1703             :   // Emit a store to the stack slot.  Use a truncstore if the input value is
    1704             :   // later than DestVT.
    1705         428 :   SDValue Store;
    1706             : 
    1707         428 :   if (SrcSize > SlotSize)
    1708           0 :     Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
    1709           0 :                               SlotVT, SrcAlign);
    1710             :   else {
    1711             :     assert(SrcSize == SlotSize && "Invalid store");
    1712         428 :     Store =
    1713        1712 :         DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
    1714             :   }
    1715             : 
    1716             :   // Result is a load from the stack slot.
    1717         428 :   if (SlotSize == DestSize)
    1718         856 :     return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
    1719             : 
    1720             :   assert(SlotSize < DestSize && "Unknown extension!");
    1721           0 :   return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
    1722           0 :                         DestAlign);
    1723             : }
    1724             : 
    1725          18 : SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
    1726          36 :   SDLoc dl(Node);
    1727             :   // Create a vector sized/aligned stack slot, store the value to element #0,
    1728             :   // then load the whole vector back out.
    1729          36 :   SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
    1730             : 
    1731          18 :   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
    1732          18 :   int SPFI = StackPtrFI->getIndex();
    1733             : 
    1734          18 :   SDValue Ch = DAG.getTruncStore(
    1735          36 :       DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
    1736          18 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
    1737          72 :       Node->getValueType(0).getVectorElementType());
    1738          18 :   return DAG.getLoad(
    1739             :       Node->getValueType(0), dl, Ch, StackPtr,
    1740          54 :       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
    1741             : }
    1742             : 
    1743             : static bool
    1744          27 : ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
    1745             :                      const TargetLowering &TLI, SDValue &Res) {
    1746          54 :   unsigned NumElems = Node->getNumOperands();
    1747          54 :   SDLoc dl(Node);
    1748          54 :   EVT VT = Node->getValueType(0);
    1749             : 
    1750             :   // Try to group the scalars into pairs, shuffle the pairs together, then
    1751             :   // shuffle the pairs of pairs together, etc. until the vector has
    1752             :   // been built. This will work only if all of the necessary shuffle masks
    1753             :   // are legal.
    1754             : 
    1755             :   // We do this in two phases; first to check the legality of the shuffles,
    1756             :   // and next, assuming that all shuffles are legal, to create the new nodes.
    1757          81 :   for (int Phase = 0; Phase < 2; ++Phase) {
    1758         108 :     SmallVector<std::pair<SDValue, SmallVector<int, 16> >, 16> IntermedVals,
    1759         108 :                                                                NewIntermedVals;
    1760         398 :     for (unsigned i = 0; i < NumElems; ++i) {
    1761         688 :       SDValue V = Node->getOperand(i);
    1762         688 :       if (V.isUndef())
    1763          16 :         continue;
    1764             : 
    1765         328 :       SDValue Vec;
    1766         328 :       if (Phase)
    1767         164 :         Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
    1768        1640 :       IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
    1769             :     }
    1770             : 
    1771         206 :     while (IntermedVals.size() > 2) {
    1772          76 :       NewIntermedVals.clear();
    1773         372 :       for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
    1774             :         // This vector and the next vector are shuffled together (simply to
    1775             :         // append the one to the other).
    1776         660 :         SmallVector<int, 16> ShuffleVec(NumElems, -1);
    1777             : 
    1778         440 :         SmallVector<int, 16> FinalIndices;
    1779         880 :         FinalIndices.reserve(IntermedVals[i].second.size() +
    1780         660 :                              IntermedVals[i+1].second.size());
    1781             : 
    1782         220 :         int k = 0;
    1783         960 :         for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
    1784             :              ++j, ++k) {
    1785         600 :           ShuffleVec[k] = j;
    1786         900 :           FinalIndices.push_back(IntermedVals[i].second[j]);
    1787             :         }
    1788         960 :         for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
    1789             :              ++j, ++k) {
    1790         600 :           ShuffleVec[k] = NumElems + j;
    1791         900 :           FinalIndices.push_back(IntermedVals[i+1].second[j]);
    1792             :         }
    1793             : 
    1794         220 :         SDValue Shuffle;
    1795         220 :         if (Phase)
    1796         220 :           Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
    1797         220 :                                          IntermedVals[i+1].first,
    1798         220 :                                          ShuffleVec);
    1799         220 :         else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
    1800           0 :           return false;
    1801         220 :         NewIntermedVals.push_back(
    1802         660 :             std::make_pair(Shuffle, std::move(FinalIndices)));
    1803             :       }
    1804             : 
    1805             :       // If we had an odd number of defined values, then append the last
    1806             :       // element to the array of new vectors.
    1807          76 :       if ((IntermedVals.size() & 1) != 0)
    1808           0 :         NewIntermedVals.push_back(IntermedVals.back());
    1809             : 
    1810          76 :       IntermedVals.swap(NewIntermedVals);
    1811             :     }
    1812             : 
    1813             :     assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
    1814             :            "Invalid number of intermediate vectors");
    1815          54 :     SDValue Vec1 = IntermedVals[0].first;
    1816          54 :     SDValue Vec2;
    1817          54 :     if (IntermedVals.size() > 1)
    1818          54 :       Vec2 = IntermedVals[1].first;
    1819           0 :     else if (Phase)
    1820           0 :       Vec2 = DAG.getUNDEF(VT);
    1821             : 
    1822         162 :     SmallVector<int, 16> ShuffleVec(NumElems, -1);
    1823         326 :     for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
    1824         656 :       ShuffleVec[IntermedVals[0].second[i]] = i;
    1825         326 :     for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
    1826         656 :       ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
    1827             : 
    1828          54 :     if (Phase)
    1829          27 :       Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
    1830          54 :     else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
    1831           0 :       return false;
    1832             :   }
    1833             : 
    1834             :   return true;
    1835             : }
    1836             : 
    1837             : /// Expand a BUILD_VECTOR node on targets that don't
    1838             : /// support the operation, but do support the resultant vector type.
    1839       17304 : SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
    1840       34608 :   unsigned NumElems = Node->getNumOperands();
    1841       17304 :   SDValue Value1, Value2;
    1842       34608 :   SDLoc dl(Node);
    1843       34608 :   EVT VT = Node->getValueType(0);
    1844       51912 :   EVT OpVT = Node->getOperand(0).getValueType();
    1845       17304 :   EVT EltVT = VT.getVectorElementType();
    1846             : 
    1847             :   // If the only non-undef value is the low element, turn this into a
    1848             :   // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
    1849       17304 :   bool isOnlyLowElement = true;
    1850       17304 :   bool MoreThanTwoValues = false;
    1851       17304 :   bool isConstant = true;
    1852      217908 :   for (unsigned i = 0; i < NumElems; ++i) {
    1853      401208 :     SDValue V = Node->getOperand(i);
    1854      401208 :     if (V.isUndef())
    1855       11461 :       continue;
    1856      189143 :     if (i > 0)
    1857      172164 :       isOnlyLowElement = false;
    1858      183825 :     if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
    1859             :       isConstant = false;
    1860             : 
    1861      189143 :     if (!Value1.getNode()) {
    1862             :       Value1 = V;
    1863      171839 :     } else if (!Value2.getNode()) {
    1864       50779 :       if (V != Value1)
    1865             :         Value2 = V;
    1866       92618 :     } else if (V != Value1 && V != Value2) {
    1867             :       MoreThanTwoValues = true;
    1868             :     }
    1869             :   }
    1870             : 
    1871       17304 :   if (!Value1.getNode())
    1872           0 :     return DAG.getUNDEF(VT);
    1873             : 
    1874       17304 :   if (isOnlyLowElement)
    1875          45 :     return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
    1876             : 
    1877             :   // If all elements are constants, create a load from the constant pool.
    1878       17289 :   if (isConstant) {
    1879       30980 :     SmallVector<Constant*, 16> CV;
    1880      207858 :     for (unsigned i = 0, e = NumElems; i != e; ++i) {
    1881             :       if (ConstantFPSDNode *V =
    1882      390048 :           dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
    1883        5312 :         CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
    1884             :       } else if (ConstantSDNode *V =
    1885      549955 :                  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
    1886      175843 :         if (OpVT==EltVT)
    1887      173245 :           CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
    1888             :         else {
    1889             :           // If OpVT and EltVT don't match, EltVT is not legal and the
    1890             :           // element values have been promoted/truncated earlier.  Undo this;
    1891             :           // we don't want a v16i8 to become a v16i32 for example.
    1892        2598 :           const ConstantInt *CI = V->getConstantIntValue();
    1893        2598 :           CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
    1894             :                                         CI->getZExtValue()));
    1895             :         }
    1896             :       } else {
    1897             :         assert(Node->getOperand(i).isUndef());
    1898       11213 :         Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
    1899       11213 :         CV.push_back(UndefValue::get(OpNTy));
    1900             :       }
    1901             :     }
    1902       15490 :     Constant *CP = ConstantVector::get(CV);
    1903             :     SDValue CPIdx =
    1904       61960 :         DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
    1905       15490 :     unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
    1906       15490 :     return DAG.getLoad(
    1907       15490 :         VT, dl, DAG.getEntryNode(), CPIdx,
    1908             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
    1909       30980 :         Alignment);
    1910             :   }
    1911             : 
    1912        3598 :   SmallSet<SDValue, 16> DefinedValues;
    1913       10003 :   for (unsigned i = 0; i < NumElems; ++i) {
    1914       24612 :     if (Node->getOperand(i).isUndef())
    1915         231 :       continue;
    1916       15946 :     DefinedValues.insert(Node->getOperand(i));
    1917             :   }
    1918             : 
    1919        3598 :   if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
    1920        1728 :     if (!MoreThanTwoValues) {
    1921        3475 :       SmallVector<int, 8> ShuffleVec(NumElems, -1);
    1922        8991 :       for (unsigned i = 0; i < NumElems; ++i) {
    1923       14580 :         SDValue V = Node->getOperand(i);
    1924       14580 :         if (V.isUndef())
    1925         198 :           continue;
    1926        8304 :         ShuffleVec[i] = V == Value1 ? 0 : NumElems;
    1927             :       }
    1928        5103 :       if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
    1929             :         // Get the splatted value into the low element of a vector register.
    1930        3256 :         SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
    1931        1628 :         SDValue Vec2;
    1932        1628 :         if (Value2.getNode())
    1933        2270 :           Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
    1934             :         else
    1935         493 :           Vec2 = DAG.getUNDEF(VT);
    1936             : 
    1937             :         // Return shuffle(LowValVec, undef, <0,0,0,0>)
    1938        3256 :         return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
    1939             :       }
    1940             :     } else {
    1941          27 :       SDValue Res;
    1942          27 :       if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
    1943          27 :         return Res;
    1944             :     }
    1945             :   }
    1946             : 
    1947             :   // Otherwise, we can't handle this case efficiently.
    1948         144 :   return ExpandVectorBuildThroughStack(Node);
    1949             : }
    1950             : 
    1951             : // Expand a node into a call to a libcall.  If the result value
    1952             : // does not fit into a register, return the lo part and set the hi part to the
    1953             : // by-reg argument.  If it does fit into a single register, return the result
    1954             : // and leave the Hi part unset.
    1955        1855 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
    1956             :                                             bool isSigned) {
    1957        3710 :   TargetLowering::ArgListTy Args;
    1958        1855 :   TargetLowering::ArgListEntry Entry;
    1959        9082 :   for (const SDValue &Op : Node->op_values()) {
    1960        2686 :     EVT ArgVT = Op.getValueType();
    1961        2686 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    1962        2686 :     Entry.Node = Op;
    1963        2686 :     Entry.Ty = ArgTy;
    1964        2686 :     Entry.IsSExt = isSigned;
    1965        2686 :     Entry.IsZExt = !isSigned;
    1966        2686 :     Args.push_back(Entry);
    1967             :   }
    1968        3710 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    1969       11130 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    1970             : 
    1971        3710 :   Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
    1972             : 
    1973             :   // By default, the input chain to this libcall is the entry node of the
    1974             :   // function. If the libcall is going to be emitted as a tail call then
    1975             :   // TLI.isUsedByReturnOnly will change it to the right chain if the return
    1976             :   // node which is being folded has a non-entry input chain.
    1977        3710 :   SDValue InChain = DAG.getEntryNode();
    1978             : 
    1979             :   // isTailCall may be true since the callee does not reference caller stack
    1980             :   // frame. Check if it's in the right position and that the return types match.
    1981        1855 :   SDValue TCChain = InChain;
    1982        1855 :   const Function *F = DAG.getMachineFunction().getFunction();
    1983             :   bool isTailCall =
    1984        2136 :       TLI.isInTailCallPosition(DAG, Node, TCChain) &&
    1985         293 :       (RetTy == F->getReturnType() || F->getReturnType()->isVoidTy());
    1986             :   if (isTailCall)
    1987         275 :     InChain = TCChain;
    1988             : 
    1989        3710 :   TargetLowering::CallLoweringInfo CLI(DAG);
    1990        5565 :   CLI.setDebugLoc(SDLoc(Node))
    1991        1855 :       .setChain(InChain)
    1992        1855 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    1993        5565 :                     std::move(Args))
    1994        3710 :       .setTailCall(isTailCall)
    1995        3710 :       .setSExtResult(isSigned)
    1996        3710 :       .setZExtResult(!isSigned)
    1997        1855 :       .setIsPostTypeLegalization(true);
    1998             : 
    1999        1855 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2000             : 
    2001        1855 :   if (!CallInfo.second.getNode())
    2002             :     // It's a tailcall, return the chain (which is the DAG root).
    2003         233 :     return DAG.getRoot();
    2004             : 
    2005        1622 :   return CallInfo.first;
    2006             : }
    2007             : 
    2008             : /// Generate a libcall taking the given operands as arguments
    2009             : /// and returning a result of type RetVT.
    2010           5 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
    2011             :                                             const SDValue *Ops, unsigned NumOps,
    2012             :                                             bool isSigned, const SDLoc &dl) {
    2013          10 :   TargetLowering::ArgListTy Args;
    2014           5 :   Args.reserve(NumOps);
    2015             : 
    2016           5 :   TargetLowering::ArgListEntry Entry;
    2017          25 :   for (unsigned i = 0; i != NumOps; ++i) {
    2018          20 :     Entry.Node = Ops[i];
    2019          40 :     Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
    2020          20 :     Entry.IsSExt = isSigned;
    2021          20 :     Entry.IsZExt = !isSigned;
    2022          20 :     Args.push_back(Entry);
    2023             :   }
    2024          10 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2025          30 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2026             : 
    2027           5 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2028             : 
    2029          10 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2030           5 :   CLI.setDebugLoc(dl)
    2031          15 :       .setChain(DAG.getEntryNode())
    2032           5 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2033          15 :                     std::move(Args))
    2034          10 :       .setSExtResult(isSigned)
    2035          10 :       .setZExtResult(!isSigned)
    2036           5 :       .setIsPostTypeLegalization(true);
    2037             : 
    2038           5 :   std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2039             : 
    2040          10 :   return CallInfo.first;
    2041             : }
    2042             : 
    2043             : // Expand a node into a call to a libcall. Similar to
    2044             : // ExpandLibCall except that the first operand is the in-chain.
    2045             : std::pair<SDValue, SDValue>
    2046         309 : SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
    2047             :                                          SDNode *Node,
    2048             :                                          bool isSigned) {
    2049         618 :   SDValue InChain = Node->getOperand(0);
    2050             : 
    2051         618 :   TargetLowering::ArgListTy Args;
    2052         309 :   TargetLowering::ArgListEntry Entry;
    2053        1253 :   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
    2054        1905 :     EVT ArgVT = Node->getOperand(i).getValueType();
    2055         635 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2056        1270 :     Entry.Node = Node->getOperand(i);
    2057         635 :     Entry.Ty = ArgTy;
    2058         635 :     Entry.IsSExt = isSigned;
    2059         635 :     Entry.IsZExt = !isSigned;
    2060         635 :     Args.push_back(Entry);
    2061             :   }
    2062         618 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2063        1854 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2064             : 
    2065         618 :   Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
    2066             : 
    2067         618 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2068         927 :   CLI.setDebugLoc(SDLoc(Node))
    2069         309 :       .setChain(InChain)
    2070         309 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2071         927 :                     std::move(Args))
    2072         618 :       .setSExtResult(isSigned)
    2073         618 :       .setZExtResult(!isSigned);
    2074             : 
    2075         309 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2076             : 
    2077         309 :   return CallInfo;
    2078             : }
    2079             : 
    2080        1250 : SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
    2081             :                                               RTLIB::Libcall Call_F32,
    2082             :                                               RTLIB::Libcall Call_F64,
    2083             :                                               RTLIB::Libcall Call_F80,
    2084             :                                               RTLIB::Libcall Call_F128,
    2085             :                                               RTLIB::Libcall Call_PPCF128) {
    2086        1250 :   if (Node->isStrictFPOpcode())
    2087          22 :     Node = DAG.mutateStrictFPToFP(Node);
    2088             : 
    2089             :   RTLIB::Libcall LC;
    2090        1250 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2091           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2092             :   case MVT::f32: LC = Call_F32; break;
    2093         431 :   case MVT::f64: LC = Call_F64; break;
    2094          25 :   case MVT::f80: LC = Call_F80; break;
    2095          35 :   case MVT::f128: LC = Call_F128; break;
    2096           0 :   case MVT::ppcf128: LC = Call_PPCF128; break;
    2097             :   }
    2098        1250 :   return ExpandLibCall(LC, Node, false);
    2099             : }
    2100             : 
    2101         116 : SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
    2102             :                                                RTLIB::Libcall Call_I8,
    2103             :                                                RTLIB::Libcall Call_I16,
    2104             :                                                RTLIB::Libcall Call_I32,
    2105             :                                                RTLIB::Libcall Call_I64,
    2106             :                                                RTLIB::Libcall Call_I128) {
    2107             :   RTLIB::Libcall LC;
    2108         116 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2109           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2110             :   case MVT::i8:   LC = Call_I8; break;
    2111          14 :   case MVT::i16:  LC = Call_I16; break;
    2112         102 :   case MVT::i32:  LC = Call_I32; break;
    2113           0 :   case MVT::i64:  LC = Call_I64; break;
    2114           0 :   case MVT::i128: LC = Call_I128; break;
    2115             :   }
    2116         116 :   return ExpandLibCall(LC, Node, isSigned);
    2117             : }
    2118             : 
    2119             : /// Issue libcalls to __{u}divmod to compute div / rem pairs.
    2120             : void
    2121          14 : SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
    2122             :                                           SmallVectorImpl<SDValue> &Results) {
    2123          28 :   unsigned Opcode = Node->getOpcode();
    2124          14 :   bool isSigned = Opcode == ISD::SDIVREM;
    2125             : 
    2126             :   RTLIB::Libcall LC;
    2127          14 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2128           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2129           0 :   case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
    2130           0 :   case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
    2131          14 :   case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
    2132           0 :   case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
    2133           0 :   case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
    2134             :   }
    2135             : 
    2136             :   // The input chain to this libcall is the entry node of the function.
    2137             :   // Legalizing the call will automatically add the previous call to the
    2138             :   // dependence.
    2139          28 :   SDValue InChain = DAG.getEntryNode();
    2140             : 
    2141          28 :   EVT RetVT = Node->getValueType(0);
    2142          14 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2143             : 
    2144          28 :   TargetLowering::ArgListTy Args;
    2145          14 :   TargetLowering::ArgListEntry Entry;
    2146          84 :   for (const SDValue &Op : Node->op_values()) {
    2147          28 :     EVT ArgVT = Op.getValueType();
    2148          28 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2149          28 :     Entry.Node = Op;
    2150          28 :     Entry.Ty = ArgTy;
    2151          28 :     Entry.IsSExt = isSigned;
    2152          28 :     Entry.IsZExt = !isSigned;
    2153          28 :     Args.push_back(Entry);
    2154             :   }
    2155             : 
    2156             :   // Also pass the return address of the remainder.
    2157          14 :   SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
    2158          14 :   Entry.Node = FIPtr;
    2159          14 :   Entry.Ty = RetTy->getPointerTo();
    2160          14 :   Entry.IsSExt = isSigned;
    2161          14 :   Entry.IsZExt = !isSigned;
    2162          14 :   Args.push_back(Entry);
    2163             : 
    2164          28 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2165          84 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2166             : 
    2167          28 :   SDLoc dl(Node);
    2168          28 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2169          14 :   CLI.setDebugLoc(dl)
    2170          14 :       .setChain(InChain)
    2171          14 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
    2172          42 :                     std::move(Args))
    2173          28 :       .setSExtResult(isSigned)
    2174          28 :       .setZExtResult(!isSigned);
    2175             : 
    2176          14 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2177             : 
    2178             :   // Remainder is loaded back from the stack frame.
    2179             :   SDValue Rem =
    2180          42 :       DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
    2181          14 :   Results.push_back(CallInfo.first);
    2182          14 :   Results.push_back(Rem);
    2183          14 : }
    2184             : 
    2185             : /// Return true if sincos libcall is available.
    2186             : static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
    2187             :   RTLIB::Libcall LC;
    2188         261 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2189           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2190             :   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
    2191          90 :   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
    2192          19 :   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
    2193          12 :   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
    2194           0 :   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
    2195             :   }
    2196         522 :   return TLI.getLibcallName(LC) != nullptr;
    2197             : }
    2198             : 
    2199             : /// Only issue sincos libcall if both sin and cos are needed.
    2200             : static bool useSinCos(SDNode *Node) {
    2201         139 :   unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
    2202         139 :     ? ISD::FCOS : ISD::FSIN;
    2203             : 
    2204         278 :   SDValue Op0 = Node->getOperand(0);
    2205         139 :   for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
    2206         617 :        UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
    2207         139 :     SDNode *User = *UI;
    2208         139 :     if (User == Node)
    2209             :       continue;
    2210             :     // The other user might have been turned into sincos already.
    2211          78 :     if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
    2212             :       return true;
    2213             :   }
    2214             :   return false;
    2215             : }
    2216             : 
    2217             : /// Issue libcalls to sincos to compute sin / cos pairs.
    2218             : void
    2219          30 : SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
    2220             :                                           SmallVectorImpl<SDValue> &Results) {
    2221             :   RTLIB::Libcall LC;
    2222          30 :   switch (Node->getSimpleValueType(0).SimpleTy) {
    2223           0 :   default: llvm_unreachable("Unexpected request for libcall!");
    2224             :   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
    2225          11 :   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
    2226           5 :   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
    2227           3 :   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
    2228           0 :   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
    2229             :   }
    2230             : 
    2231             :   // The input chain to this libcall is the entry node of the function.
    2232             :   // Legalizing the call will automatically add the previous call to the
    2233             :   // dependence.
    2234          60 :   SDValue InChain = DAG.getEntryNode();
    2235             : 
    2236          60 :   EVT RetVT = Node->getValueType(0);
    2237          30 :   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
    2238             : 
    2239          60 :   TargetLowering::ArgListTy Args;
    2240          30 :   TargetLowering::ArgListEntry Entry;
    2241             : 
    2242             :   // Pass the argument.
    2243          60 :   Entry.Node = Node->getOperand(0);
    2244          30 :   Entry.Ty = RetTy;
    2245             :   Entry.IsSExt = false;
    2246             :   Entry.IsZExt = false;
    2247          30 :   Args.push_back(Entry);
    2248             : 
    2249             :   // Pass the return address of sin.
    2250          30 :   SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
    2251          30 :   Entry.Node = SinPtr;
    2252          30 :   Entry.Ty = RetTy->getPointerTo();
    2253          30 :   Entry.IsSExt = false;
    2254          30 :   Entry.IsZExt = false;
    2255          30 :   Args.push_back(Entry);
    2256             : 
    2257             :   // Also pass the return address of the cos.
    2258          30 :   SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
    2259          30 :   Entry.Node = CosPtr;
    2260          30 :   Entry.Ty = RetTy->getPointerTo();
    2261          30 :   Entry.IsSExt = false;
    2262          30 :   Entry.IsZExt = false;
    2263          30 :   Args.push_back(Entry);
    2264             : 
    2265          60 :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
    2266         180 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    2267             : 
    2268          60 :   SDLoc dl(Node);
    2269          60 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2270          90 :   CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
    2271          60 :       TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
    2272         120 :       std::move(Args));
    2273             : 
    2274          30 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2275             : 
    2276          60 :   Results.push_back(
    2277         120 :       DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
    2278          30 :   Results.push_back(
    2279         120 :       DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
    2280          30 : }
    2281             : 
    2282             : /// This function is responsible for legalizing a
    2283             : /// INT_TO_FP operation of the specified operand when the target requests that
    2284             : /// we expand it.  At this point, we know that the result and operand types are
    2285             : /// legal for the target.
    2286         138 : SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
    2287             :                                                    EVT DestVT,
    2288             :                                                    const SDLoc &dl) {
    2289             :   // TODO: Should any fast-math-flags be set for the created nodes?
    2290             : 
    2291         428 :   if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
    2292             :     // simple 32-bit [signed|unsigned] integer to float/double expansion
    2293             : 
    2294             :     // Get the stack frame index of a 8 byte buffer.
    2295         100 :     SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
    2296             : 
    2297             :     // word offset constant for Hi/Lo address computation
    2298          50 :     SDValue WordOff = DAG.getConstant(sizeof(int), dl,
    2299         100 :                                       StackSlot.getValueType());
    2300             :     // set up Hi and Lo (into buffer) address based on endian
    2301          50 :     SDValue Hi = StackSlot;
    2302          50 :     SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
    2303         100 :                              StackSlot, WordOff);
    2304         100 :     if (DAG.getDataLayout().isLittleEndian())
    2305             :       std::swap(Hi, Lo);
    2306             : 
    2307             :     // if signed map to unsigned space
    2308          50 :     SDValue Op0Mapped;
    2309          50 :     if (isSigned) {
    2310             :       // constant used to invert sign bit (signed to unsigned mapping)
    2311           8 :       SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
    2312          12 :       Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
    2313             :     } else {
    2314          46 :       Op0Mapped = Op0;
    2315             :     }
    2316             :     // store the lo of the constructed double - based on integer input
    2317         100 :     SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
    2318         150 :                                   MachinePointerInfo());
    2319             :     // initial hi portion of constructed double
    2320         100 :     SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
    2321             :     // store the hi of the constructed double - biased exponent
    2322             :     SDValue Store2 =
    2323         150 :         DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
    2324             :     // load the constructed double
    2325             :     SDValue Load =
    2326         200 :         DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
    2327             :     // FP constant to bias correct the final result
    2328          50 :     SDValue Bias = DAG.getConstantFP(isSigned ?
    2329             :                                      BitsToDouble(0x4330000080000000ULL) :
    2330             :                                      BitsToDouble(0x4330000000000000ULL),
    2331          50 :                                      dl, MVT::f64);
    2332             :     // subtract the bias
    2333         150 :     SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
    2334             :     // final result
    2335          50 :     SDValue Result;
    2336             :     // handle final rounding
    2337         100 :     if (DestVT == MVT::f64) {
    2338             :       // do nothing
    2339          10 :       Result = Sub;
    2340          40 :     } else if (DestVT.bitsLT(MVT::f64)) {
    2341          72 :       Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
    2342          72 :                            DAG.getIntPtrConstant(0, dl));
    2343           4 :     } else if (DestVT.bitsGT(MVT::f64)) {
    2344           8 :       Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
    2345             :     }
    2346          50 :     return Result;
    2347             :   }
    2348             :   assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
    2349             :   // Code below here assumes !isSigned without checking again.
    2350             : 
    2351             :   // Implementation of unsigned i64 to f64 following the algorithm in
    2352             :   // __floatundidf in compiler_rt. This implementation has the advantage
    2353             :   // of performing rounding correctly, both in the default rounding mode
    2354             :   // and in all alternate rounding modes.
    2355             :   // TODO: Generalize this for use with other types.
    2356         263 :   if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
    2357             :     SDValue TwoP52 =
    2358           8 :       DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
    2359             :     SDValue TwoP84PlusTwoP52 =
    2360           4 :       DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
    2361           4 :                         MVT::f64);
    2362             :     SDValue TwoP84 =
    2363           8 :       DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
    2364             : 
    2365           8 :     SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
    2366           4 :     SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
    2367          12 :                              DAG.getConstant(32, dl, MVT::i64));
    2368          12 :     SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
    2369          12 :     SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
    2370          12 :     SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
    2371          12 :     SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
    2372           4 :     SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
    2373           8 :                                 TwoP84PlusTwoP52);
    2374          12 :     return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
    2375             :   }
    2376             : 
    2377             :   // Implementation of unsigned i64 to f32.
    2378             :   // TODO: Generalize this for use with other types.
    2379         251 :   if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
    2380             :     // For unsigned conversions, convert them to signed conversions using the
    2381             :     // algorithm from the x86_64 __floatundidf in compiler_rt.
    2382          82 :     if (!isSigned) {
    2383         246 :       SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
    2384             : 
    2385          82 :       SDValue ShiftConst = DAG.getConstant(
    2386         246 :           1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
    2387         246 :       SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
    2388         164 :       SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
    2389         246 :       SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
    2390         246 :       SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
    2391             : 
    2392         246 :       SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
    2393         246 :       SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
    2394             : 
    2395             :       // TODO: This really should be implemented using a branch rather than a
    2396             :       // select.  We happen to get lucky and machinesink does the right
    2397             :       // thing most of the time.  This would be a good candidate for a
    2398             :       //pseudo-op, or, even better, for whole-function isel.
    2399         164 :       SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
    2400         246 :         Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
    2401         164 :       return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
    2402             :     }
    2403             : 
    2404             :     // Otherwise, implement the fully general conversion.
    2405             : 
    2406           0 :     SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
    2407           0 :          DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
    2408           0 :     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
    2409           0 :          DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
    2410           0 :     SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
    2411           0 :          DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
    2412           0 :     SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
    2413           0 :                               DAG.getConstant(UINT64_C(0), dl, MVT::i64),
    2414           0 :                               ISD::SETNE);
    2415           0 :     SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
    2416           0 :     SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
    2417           0 :                               DAG.getConstant(UINT64_C(0x0020000000000000), dl,
    2418           0 :                                               MVT::i64),
    2419           0 :                               ISD::SETUGE);
    2420           0 :     SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
    2421           0 :     EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
    2422             : 
    2423           0 :     SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
    2424           0 :                              DAG.getConstant(32, dl, SHVT));
    2425           0 :     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
    2426           0 :     SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
    2427             :     SDValue TwoP32 =
    2428           0 :       DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
    2429           0 :                         MVT::f64);
    2430           0 :     SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
    2431           0 :     SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
    2432           0 :     SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
    2433           0 :     SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
    2434           0 :     return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
    2435           0 :                        DAG.getIntPtrConstant(0, dl));
    2436             :   }
    2437             : 
    2438           4 :   SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
    2439             : 
    2440           2 :   SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
    2441             :                                  Op0,
    2442             :                                  DAG.getConstant(0, dl, Op0.getValueType()),
    2443           6 :                                  ISD::SETLT);
    2444           2 :   SDValue Zero = DAG.getIntPtrConstant(0, dl),
    2445           2 :           Four = DAG.getIntPtrConstant(4, dl);
    2446           2 :   SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
    2447           4 :                                     SignSet, Four, Zero);
    2448             : 
    2449             :   // If the sign bit of the integer is set, the large number will be treated
    2450             :   // as a negative number.  To counteract this, the dynamic code adds an
    2451             :   // offset depending on the data type.
    2452             :   uint64_t FF;
    2453           2 :   switch (Op0.getSimpleValueType().SimpleTy) {
    2454           0 :   default: llvm_unreachable("Unsupported integer type!");
    2455             :   case MVT::i8 : FF = 0x43800000ULL; break;  // 2^8  (as a float)
    2456           0 :   case MVT::i16: FF = 0x47800000ULL; break;  // 2^16 (as a float)
    2457           1 :   case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
    2458           1 :   case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
    2459             :   }
    2460           4 :   if (DAG.getDataLayout().isLittleEndian())
    2461           0 :     FF <<= 32;
    2462           2 :   Constant *FudgeFactor = ConstantInt::get(
    2463           4 :                                        Type::getInt64Ty(*DAG.getContext()), FF);
    2464             : 
    2465             :   SDValue CPIdx =
    2466           8 :       DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
    2467           2 :   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
    2468           6 :   CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
    2469           4 :   Alignment = std::min(Alignment, 4u);
    2470           2 :   SDValue FudgeInReg;
    2471           4 :   if (DestVT == MVT::f32)
    2472           2 :     FudgeInReg = DAG.getLoad(
    2473           1 :         MVT::f32, dl, DAG.getEntryNode(), CPIdx,
    2474             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
    2475           4 :         Alignment);
    2476             :   else {
    2477           1 :     SDValue Load = DAG.getExtLoad(
    2478           1 :         ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
    2479             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
    2480           3 :         Alignment);
    2481           2 :     HandleSDNode Handle(Load);
    2482           1 :     LegalizeOp(Load.getNode());
    2483           1 :     FudgeInReg = Handle.getValue();
    2484             :   }
    2485             : 
    2486           4 :   return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
    2487             : }
    2488             : 
    2489             : /// This function is responsible for legalizing a
    2490             : /// *INT_TO_FP operation of the specified operand when the target requests that
    2491             : /// we promote it.  At this point, we know that the result and operand types are
    2492             : /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
    2493             : /// operation that takes a larger input.
    2494          70 : SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
    2495             :                                                     bool isSigned,
    2496             :                                                     const SDLoc &dl) {
    2497             :   // First step, figure out the appropriate *INT_TO_FP operation to use.
    2498         140 :   EVT NewInTy = LegalOp.getValueType();
    2499             : 
    2500          70 :   unsigned OpToUse = 0;
    2501             : 
    2502             :   // Scan for the appropriate larger type to use.
    2503             :   while (1) {
    2504         162 :     NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
    2505             :     assert(NewInTy.isInteger() && "Ran out of possibilities!");
    2506             : 
    2507             :     // If the target supports SINT_TO_FP of this type, use it.
    2508          81 :     if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
    2509             :       OpToUse = ISD::SINT_TO_FP;
    2510             :       break;
    2511             :     }
    2512          11 :     if (isSigned) continue;
    2513             : 
    2514             :     // If the target supports UINT_TO_FP of this type, use it.
    2515           1 :     if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
    2516             :       OpToUse = ISD::UINT_TO_FP;
    2517             :       break;
    2518             :     }
    2519             : 
    2520             :     // Otherwise, try a larger type.
    2521             :   }
    2522             : 
    2523             :   // Okay, we found the operation and type to use.  Zero extend our input to the
    2524             :   // desired type then run the operation on it.
    2525          70 :   return DAG.getNode(OpToUse, dl, DestVT,
    2526          70 :                      DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
    2527         140 :                                  dl, NewInTy, LegalOp));
    2528             : }
    2529             : 
    2530             : /// This function is responsible for legalizing a
    2531             : /// FP_TO_*INT operation of the specified operand when the target requests that
    2532             : /// we promote it.  At this point, we know that the result and operand types are
    2533             : /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
    2534             : /// operation that returns a larger result.
    2535         108 : SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
    2536             :                                                     bool isSigned,
    2537             :                                                     const SDLoc &dl) {
    2538             :   // First step, figure out the appropriate FP_TO*INT operation to use.
    2539         108 :   EVT NewOutTy = DestVT;
    2540             : 
    2541         108 :   unsigned OpToUse = 0;
    2542             : 
    2543             :   // Scan for the appropriate larger type to use.
    2544             :   while (1) {
    2545         256 :     NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
    2546             :     assert(NewOutTy.isInteger() && "Ran out of possibilities!");
    2547             : 
    2548             :     // A larger signed type can hold all unsigned values of the requested type,
    2549             :     // so using FP_TO_SINT is valid
    2550         128 :     if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
    2551             :       OpToUse = ISD::FP_TO_SINT;
    2552             :       break;
    2553             :     }
    2554             : 
    2555             :     // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
    2556          20 :     if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
    2557             :       OpToUse = ISD::FP_TO_UINT;
    2558             :       break;
    2559             :     }
    2560             : 
    2561             :     // Otherwise, try a larger type.
    2562             :   }
    2563             : 
    2564             : 
    2565             :   // Okay, we found the operation and type to use.
    2566         216 :   SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
    2567             : 
    2568             :   // Truncate the result of the extended FP_TO_*INT operation to the desired
    2569             :   // size.
    2570         216 :   return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
    2571             : }
    2572             : 
    2573             : /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
    2574          92 : SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
    2575         184 :   EVT VT = Op.getValueType();
    2576         184 :   EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2577          92 :   unsigned Sz = VT.getScalarSizeInBits();
    2578             : 
    2579          92 :   SDValue Tmp, Tmp2, Tmp3;
    2580             : 
    2581             :   // If we can, perform BSWAP first and then the mask+swap the i4, then i2
    2582             :   // and finally the i1 pairs.
    2583             :   // TODO: We can easily support i4/i2 legal types if any target ever does.
    2584         184 :   if (Sz >= 8 && isPowerOf2_32(Sz)) {
    2585             :     // Create the masks - repeating the pattern every byte.
    2586         552 :     APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
    2587         460 :     APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
    2588         756 :     for (unsigned J = 0; J != Sz; J += 8) {
    2589        1660 :       MaskHi4 = MaskHi4 | (0xF0ull << J);
    2590        1660 :       MaskLo4 = MaskLo4 | (0x0Full << J);
    2591        1660 :       MaskHi2 = MaskHi2 | (0xCCull << J);
    2592        1660 :       MaskLo2 = MaskLo2 | (0x33ull << J);
    2593        1660 :       MaskHi1 = MaskHi1 | (0xAAull << J);
    2594        1660 :       MaskLo1 = MaskLo1 | (0x55ull << J);
    2595             :     }
    2596             : 
    2597             :     // BSWAP if the type is wider than a single byte.
    2598         144 :     Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
    2599             : 
    2600             :     // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
    2601         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
    2602         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
    2603         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
    2604         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
    2605         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2606             : 
    2607             :     // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
    2608         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
    2609         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
    2610         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
    2611         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
    2612         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2613             : 
    2614             :     // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
    2615         184 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
    2616         184 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
    2617         184 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
    2618         184 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
    2619         184 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    2620          92 :     return Tmp;
    2621             :   }
    2622             : 
    2623           0 :   Tmp = DAG.getConstant(0, dl, VT);
    2624           0 :   for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
    2625           0 :     if (I < J)
    2626           0 :       Tmp2 =
    2627           0 :           DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
    2628             :     else
    2629           0 :       Tmp2 =
    2630           0 :           DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
    2631             : 
    2632           0 :     APInt Shift(Sz, 1);
    2633           0 :     Shift <<= J;
    2634           0 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
    2635           0 :     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
    2636             :   }
    2637             : 
    2638           0 :   return Tmp;
    2639             : }
    2640             : 
    2641             : /// Open code the operations for BSWAP of the specified operation.
    2642         123 : SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
    2643         246 :   EVT VT = Op.getValueType();
    2644         246 :   EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2645         123 :   SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
    2646         246 :   switch (VT.getSimpleVT().getScalarType().SimpleTy) {
    2647           0 :   default: llvm_unreachable("Unhandled Expand type in BSWAP!");
    2648          60 :   case MVT::i16:
    2649         120 :     Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2650         120 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2651         120 :     return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
    2652          25 :   case MVT::i32:
    2653          50 :     Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2654          50 :     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2655          50 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2656          50 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2657          50 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
    2658          50 :                        DAG.getConstant(0xFF0000, dl, VT));
    2659          50 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
    2660          50 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    2661          50 :     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    2662          50 :     return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
    2663          38 :   case MVT::i64:
    2664          76 :     Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    2665          76 :     Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
    2666          76 :     Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2667          76 :     Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2668          76 :     Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    2669          76 :     Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    2670          76 :     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
    2671          76 :     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    2672          76 :     Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
    2673          76 :                        DAG.getConstant(255ULL<<48, dl, VT));
    2674          76 :     Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
    2675          76 :                        DAG.getConstant(255ULL<<40, dl, VT));
    2676          76 :     Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
    2677          76 :                        DAG.getConstant(255ULL<<32, dl, VT));
    2678          76 :     Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
    2679          76 :                        DAG.getConstant(255ULL<<24, dl, VT));
    2680          76 :     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
    2681          76 :                        DAG.getConstant(255ULL<<16, dl, VT));
    2682          76 :     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
    2683          76 :                        DAG.getConstant(255ULL<<8 , dl, VT));
    2684          76 :     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
    2685          76 :     Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
    2686          76 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    2687          76 :     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    2688          76 :     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
    2689          76 :     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
    2690          76 :     return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
    2691             :   }
    2692             : }
    2693             : 
    2694             : /// Expand the specified bitcount instruction into operations.
    2695         307 : SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
    2696             :                                              const SDLoc &dl) {
    2697         307 :   switch (Opc) {
    2698           0 :   default: llvm_unreachable("Cannot expand this yet!");
    2699          65 :   case ISD::CTPOP: {
    2700         130 :     EVT VT = Op.getValueType();
    2701         130 :     EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2702          65 :     unsigned Len = VT.getSizeInBits();
    2703             : 
    2704             :     assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
    2705             :            "CTPOP not implemented for this type.");
    2706             : 
    2707             :     // This is the "best" algorithm from
    2708             :     // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
    2709             : 
    2710         260 :     SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
    2711          65 :                                      dl, VT);
    2712         260 :     SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
    2713          65 :                                      dl, VT);
    2714         260 :     SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
    2715          65 :                                      dl, VT);
    2716         260 :     SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
    2717          65 :                                      dl, VT);
    2718             : 
    2719             :     // v = v - ((v >> 1) & 0x55555555...)
    2720         130 :     Op = DAG.getNode(ISD::SUB, dl, VT, Op,
    2721          65 :                      DAG.getNode(ISD::AND, dl, VT,
    2722          65 :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2723             :                                              DAG.getConstant(1, dl, ShVT)),
    2724         260 :                                  Mask55));
    2725             :     // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
    2726         130 :     Op = DAG.getNode(ISD::ADD, dl, VT,
    2727          65 :                      DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
    2728          65 :                      DAG.getNode(ISD::AND, dl, VT,
    2729          65 :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2730             :                                              DAG.getConstant(2, dl, ShVT)),
    2731         325 :                                  Mask33));
    2732             :     // v = (v + (v >> 4)) & 0x0F0F0F0F...
    2733         130 :     Op = DAG.getNode(ISD::AND, dl, VT,
    2734          65 :                      DAG.getNode(ISD::ADD, dl, VT, Op,
    2735          65 :                                  DAG.getNode(ISD::SRL, dl, VT, Op,
    2736             :                                              DAG.getConstant(4, dl, ShVT))),
    2737         260 :                      Mask0F);
    2738             :     // v = (v * 0x01010101...) >> (Len - 8)
    2739         130 :     Op = DAG.getNode(ISD::SRL, dl, VT,
    2740          65 :                      DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
    2741         195 :                      DAG.getConstant(Len - 8, dl, ShVT));
    2742             : 
    2743          65 :     return Op;
    2744             :   }
    2745         100 :   case ISD::CTLZ_ZERO_UNDEF:
    2746             :     // This trivially expands to CTLZ.
    2747         300 :     return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
    2748          61 :   case ISD::CTLZ: {
    2749         122 :     EVT VT = Op.getValueType();
    2750          61 :     unsigned len = VT.getSizeInBits();
    2751             : 
    2752          61 :     if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
    2753          40 :       EVT SetCCVT = getSetCCResultType(VT);
    2754          80 :       SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
    2755          40 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    2756          40 :       SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
    2757          40 :       return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
    2758          40 :                          DAG.getConstant(len, dl, VT), CTLZ);
    2759             :     }
    2760             : 
    2761             :     // for now, we do this:
    2762             :     // x = x | (x >> 1);
    2763             :     // x = x | (x >> 2);
    2764             :     // ...
    2765             :     // x = x | (x >>16);
    2766             :     // x = x | (x >>32); // for 64-bit input
    2767             :     // return popcount(~x);
    2768             :     //
    2769             :     // Ref: "Hacker's Delight" by Henry Warren
    2770          42 :     EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2771         132 :     for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
    2772         111 :       SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
    2773         222 :       Op = DAG.getNode(ISD::OR, dl, VT, Op,
    2774         333 :                        DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
    2775             :     }
    2776          21 :     Op = DAG.getNOT(dl, Op, VT);
    2777          42 :     return DAG.getNode(ISD::CTPOP, dl, VT, Op);
    2778             :   }
    2779          68 :   case ISD::CTTZ_ZERO_UNDEF:
    2780             :     // This trivially expands to CTTZ.
    2781         204 :     return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
    2782          13 :   case ISD::CTTZ: {
    2783             :     // for now, we use: { return popcount(~x & (x - 1)); }
    2784             :     // unless the target has ctlz but not ctpop, in which case we use:
    2785             :     // { return 32 - nlz(~x & (x-1)); }
    2786             :     // Ref: "Hacker's Delight" by Henry Warren
    2787          26 :     EVT VT = Op.getValueType();
    2788          13 :     SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
    2789          13 :                                DAG.getNOT(dl, Op, VT),
    2790          13 :                                DAG.getNode(ISD::SUB, dl, VT, Op,
    2791          39 :                                            DAG.getConstant(1, dl, VT)));
    2792             :     // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
    2793          13 :     if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
    2794           5 :         TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
    2795           5 :       return DAG.getNode(ISD::SUB, dl, VT,
    2796           5 :                          DAG.getConstant(VT.getSizeInBits(), dl, VT),
    2797          15 :                          DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
    2798          16 :     return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
    2799             :   }
    2800             :   }
    2801             : }
    2802             : 
    2803      228092 : bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
    2804      456184 :   SmallVector<SDValue, 8> Results;
    2805      456184 :   SDLoc dl(Node);
    2806      228092 :   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
    2807             :   bool NeedInvert;
    2808      456184 :   switch (Node->getOpcode()) {
    2809         307 :   case ISD::CTPOP:
    2810             :   case ISD::CTLZ:
    2811             :   case ISD::CTLZ_ZERO_UNDEF:
    2812             :   case ISD::CTTZ:
    2813             :   case ISD::CTTZ_ZERO_UNDEF:
    2814         614 :     Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
    2815         307 :     Results.push_back(Tmp1);
    2816         307 :     break;
    2817          92 :   case ISD::BITREVERSE:
    2818         184 :     Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
    2819          92 :     break;
    2820         123 :   case ISD::BSWAP:
    2821         246 :     Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
    2822         123 :     break;
    2823           0 :   case ISD::FRAMEADDR:
    2824             :   case ISD::RETURNADDR:
    2825             :   case ISD::FRAME_TO_ARGS_OFFSET:
    2826           0 :     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
    2827           0 :     break;
    2828           6 :   case ISD::EH_DWARF_CFA: {
    2829          18 :     SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
    2830          30 :                                         TLI.getPointerTy(DAG.getDataLayout()));
    2831           6 :     SDValue Offset = DAG.getNode(ISD::ADD, dl,
    2832             :                                  CfaArg.getValueType(),
    2833             :                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
    2834             :                                              CfaArg.getValueType()),
    2835          18 :                                  CfaArg);
    2836           6 :     SDValue FA = DAG.getNode(
    2837          12 :         ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
    2838          54 :         DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
    2839          12 :     Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
    2840          12 :                                   FA, Offset));
    2841             :     break;
    2842             :   }
    2843           1 :   case ISD::FLT_ROUNDS_:
    2844           2 :     Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
    2845           1 :     break;
    2846         249 :   case ISD::EH_RETURN:
    2847             :   case ISD::EH_LABEL:
    2848             :   case ISD::PREFETCH:
    2849             :   case ISD::VAEND:
    2850             :   case ISD::EH_SJLJ_LONGJMP:
    2851             :     // If the target didn't expand these, there's nothing to do, so just
    2852             :     // preserve the chain and be done.
    2853         498 :     Results.push_back(Node->getOperand(0));
    2854         249 :     break;
    2855           3 :   case ISD::READCYCLECOUNTER:
    2856             :     // If the target didn't expand this, just return 'zero' and preserve the
    2857             :     // chain.
    2858           6 :     Results.append(Node->getNumValues() - 1,
    2859           9 :                    DAG.getConstant(0, dl, Node->getValueType(0)));
    2860           6 :     Results.push_back(Node->getOperand(0));
    2861           3 :     break;
    2862           0 :   case ISD::EH_SJLJ_SETJMP:
    2863             :     // If the target didn't expand this, just return 'zero' and preserve the
    2864             :     // chain.
    2865           0 :     Results.push_back(DAG.getConstant(0, dl, MVT::i32));
    2866           0 :     Results.push_back(Node->getOperand(0));
    2867           0 :     break;
    2868           6 :   case ISD::ATOMIC_LOAD: {
    2869             :     // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
    2870          12 :     SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
    2871          18 :     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
    2872           6 :     SDValue Swap = DAG.getAtomicCmpSwap(
    2873           6 :         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
    2874          18 :         Node->getOperand(0), Node->getOperand(1), Zero, Zero,
    2875          12 :         cast<AtomicSDNode>(Node)->getMemOperand());
    2876          12 :     Results.push_back(Swap.getValue(0));
    2877          12 :     Results.push_back(Swap.getValue(1));
    2878             :     break;
    2879             :   }
    2880          14 :   case ISD::ATOMIC_STORE: {
    2881             :     // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
    2882          14 :     SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
    2883          14 :                                  cast<AtomicSDNode>(Node)->getMemoryVT(),
    2884          28 :                                  Node->getOperand(0),
    2885          42 :                                  Node->getOperand(1), Node->getOperand(2),
    2886          28 :                                  cast<AtomicSDNode>(Node)->getMemOperand());
    2887          28 :     Results.push_back(Swap.getValue(1));
    2888             :     break;
    2889             :   }
    2890         846 :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
    2891             :     // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
    2892             :     // splits out the success value as a comparison. Expanding the resulting
    2893             :     // ATOMIC_CMP_SWAP will produce a libcall.
    2894        2538 :     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
    2895         846 :     SDValue Res = DAG.getAtomicCmpSwap(
    2896         846 :         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
    2897        3384 :         Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
    2898        2538 :         Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
    2899             : 
    2900         846 :     SDValue ExtRes = Res;
    2901         846 :     SDValue LHS = Res;
    2902        1692 :     SDValue RHS = Node->getOperand(1);
    2903             : 
    2904         846 :     EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
    2905        1692 :     EVT OuterType = Node->getValueType(0);
    2906         846 :     switch (TLI.getExtendForAtomicOps()) {
    2907          55 :     case ISD::SIGN_EXTEND:
    2908         110 :       LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
    2909         110 :                         DAG.getValueType(AtomicType));
    2910         110 :       RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
    2911         165 :                         Node->getOperand(2), DAG.getValueType(AtomicType));
    2912          55 :       ExtRes = LHS;
    2913          55 :       break;
    2914         756 :     case ISD::ZERO_EXTEND:
    2915        1512 :       LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
    2916        1512 :                         DAG.getValueType(AtomicType));
    2917        2268 :       RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
    2918         756 :       ExtRes = LHS;
    2919         756 :       break;
    2920          35 :     case ISD::ANY_EXTEND:
    2921          35 :       LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
    2922         105 :       RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
    2923          35 :       break;
    2924           0 :     default:
    2925           0 :       llvm_unreachable("Invalid atomic op extension");
    2926             :     }
    2927             : 
    2928             :     SDValue Success =
    2929        1692 :         DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
    2930             : 
    2931        1692 :     Results.push_back(ExtRes.getValue(0));
    2932         846 :     Results.push_back(Success);
    2933        1692 :     Results.push_back(Res.getValue(1));
    2934             :     break;
    2935             :   }
    2936         140 :   case ISD::DYNAMIC_STACKALLOC:
    2937         140 :     ExpandDYNAMIC_STACKALLOC(Node, Results);
    2938         140 :     break;
    2939             :   case ISD::MERGE_VALUES:
    2940     1148521 :     for (unsigned i = 0; i < Node->getNumValues(); i++)
    2941      430687 :       Results.push_back(Node->getOperand(i));
    2942             :     break;
    2943         448 :   case ISD::UNDEF: {
    2944         896 :     EVT VT = Node->getValueType(0);
    2945         448 :     if (VT.isInteger())
    2946         330 :       Results.push_back(DAG.getConstant(0, dl, VT));
    2947             :     else {
    2948             :       assert(VT.isFloatingPoint() && "Unknown value type!");
    2949         118 :       Results.push_back(DAG.getConstantFP(0, dl, VT));
    2950             :     }
    2951             :     break;
    2952             :   }
    2953         428 :   case ISD::FP_ROUND:
    2954             :   case ISD::BITCAST:
    2955         856 :     Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
    2956         856 :                             Node->getValueType(0), dl);
    2957         428 :     Results.push_back(Tmp1);
    2958         428 :     break;
    2959           0 :   case ISD::FP_EXTEND:
    2960           0 :     Tmp1 = EmitStackConvert(Node->getOperand(0),
    2961           0 :                             Node->getOperand(0).getValueType(),
    2962           0 :                             Node->getValueType(0), dl);
    2963           0 :     Results.push_back(Tmp1);
    2964           0 :     break;
    2965         755 :   case ISD::SIGN_EXTEND_INREG: {
    2966        2265 :     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
    2967        1510 :     EVT VT = Node->getValueType(0);
    2968             : 
    2969             :     // An in-register sign-extend of a boolean is a negation:
    2970             :     // 'true' (1) sign-extended is -1.
    2971             :     // 'false' (0) sign-extended is 0.
    2972             :     // However, we must mask the high bits of the source operand because the
    2973             :     // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
    2974             : 
    2975             :     // TODO: Do this for vectors too?
    2976         755 :     if (ExtraVT.getSizeInBits() == 1) {
    2977         243 :       SDValue One = DAG.getConstant(1, dl, VT);
    2978         729 :       SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
    2979         243 :       SDValue Zero = DAG.getConstant(0, dl, VT);
    2980         486 :       SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
    2981         243 :       Results.push_back(Neg);
    2982             :       break;
    2983             :     }
    2984             : 
    2985             :     // NOTE: we could fall back on load/store here too for targets without
    2986             :     // SRA.  However, it is doubtful that any exist.
    2987        1024 :     EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2988         512 :     unsigned BitsDiff = VT.getScalarSizeInBits() -
    2989         512 :                         ExtraVT.getScalarSizeInBits();
    2990         512 :     SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
    2991        1024 :     Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
    2992        2048 :                        Node->getOperand(0), ShiftCst);
    2993        1536 :     Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
    2994         512 :     Results.push_back(Tmp1);
    2995         512 :     break;
    2996             :   }
    2997           0 :   case ISD::FP_ROUND_INREG: {
    2998             :     // The only way we can lower this is to turn it into a TRUNCSTORE,
    2999             :     // EXTLOAD pair, targeting a temporary location (a stack slot).
    3000             : 
    3001             :     // NOTE: there is a choice here between constantly creating new stack
    3002             :     // slots and always reusing the same one.  We currently always create
    3003             :     // new ones, as reuse may inhibit scheduling.
    3004           0 :     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
    3005           0 :     Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
    3006           0 :                             Node->getValueType(0), dl);
    3007           0 :     Results.push_back(Tmp1);
    3008             :     break;
    3009             :   }
    3010         138 :   case ISD::SINT_TO_FP:
    3011             :   case ISD::UINT_TO_FP:
    3012         276 :     Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
    3013         552 :                                 Node->getOperand(0), Node->getValueType(0), dl);
    3014         138 :     Results.push_back(Tmp1);
    3015         138 :     break;
    3016          60 :   case ISD::FP_TO_SINT:
    3017          60 :     if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
    3018          60 :       Results.push_back(Tmp1);
    3019             :     break;
    3020         192 :   case ISD::FP_TO_UINT: {
    3021         192 :     SDValue True, False;
    3022         576 :     EVT VT =  Node->getOperand(0).getValueType();
    3023         384 :     EVT NVT = Node->getValueType(0);
    3024             :     APFloat apf(DAG.EVTToAPFloatSemantics(VT),
    3025         768 :                 APInt::getNullValue(VT.getSizeInBits()));
    3026         576 :     APInt x = APInt::getSignMask(NVT.getSizeInBits());
    3027         192 :     (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
    3028         192 :     Tmp1 = DAG.getConstantFP(apf, dl, VT);
    3029         384 :     Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
    3030         384 :                         Node->getOperand(0),
    3031         192 :                         Tmp1, ISD::SETLT);
    3032         576 :     True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
    3033             :     // TODO: Should any fast-math-flags be set for the FSUB?
    3034         384 :     False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
    3035         192 :                         DAG.getNode(ISD::FSUB, dl, VT,
    3036         768 :                                     Node->getOperand(0), Tmp1));
    3037         384 :     False = DAG.getNode(ISD::XOR, dl, NVT, False,
    3038         384 :                         DAG.getConstant(x, dl, NVT));
    3039         192 :     Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
    3040         192 :     Results.push_back(Tmp1);
    3041             :     break;
    3042             :   }
    3043          24 :   case ISD::VAARG:
    3044          24 :     Results.push_back(DAG.expandVAArg(Node));
    3045          48 :     Results.push_back(Results[0].getValue(1));
    3046          24 :     break;
    3047           1 :   case ISD::VACOPY:
    3048           1 :     Results.push_back(DAG.expandVACopy(Node));
    3049           1 :     break;
    3050        2117 :   case ISD::EXTRACT_VECTOR_ELT:
    3051        6351 :     if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
    3052             :       // This must be an access of the only element.  Return it.
    3053         112 :       Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
    3054         224 :                          Node->getOperand(0));
    3055             :     else
    3056        2061 :       Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
    3057        2117 :     Results.push_back(Tmp1);
    3058        2117 :     break;
    3059           0 :   case ISD::EXTRACT_SUBVECTOR:
    3060           0 :     Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
    3061           0 :     break;
    3062           0 :   case ISD::INSERT_SUBVECTOR:
    3063           0 :     Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
    3064           0 :     break;
    3065           0 :   case ISD::CONCAT_VECTORS: {
    3066           0 :     Results.push_back(ExpandVectorBuildThroughStack(Node));
    3067           0 :     break;
    3068             :   }
    3069          18 :   case ISD::SCALAR_TO_VECTOR:
    3070          18 :     Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
    3071          18 :     break;
    3072         386 :   case ISD::INSERT_VECTOR_ELT:
    3073         772 :     Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
    3074         772 :                                               Node->getOperand(1),
    3075         772 :                                               Node->getOperand(2), dl));
    3076         386 :     break;
    3077         181 :   case ISD::VECTOR_SHUFFLE: {
    3078         362 :     SmallVector<int, 32> NewMask;
    3079         362 :     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
    3080             : 
    3081         181 :     EVT VT = Node->getValueType(0);
    3082         181 :     EVT EltVT = VT.getVectorElementType();
    3083         362 :     SDValue Op0 = Node->getOperand(0);
    3084         362 :     SDValue Op1 = Node->getOperand(1);
    3085         181 :     if (!TLI.isTypeLegal(EltVT)) {
    3086             : 
    3087          22 :       EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
    3088             : 
    3089             :       // BUILD_VECTOR operands are allowed to be wider than the element type.
    3090             :       // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
    3091             :       // it.
    3092          11 :       if (NewEltVT.bitsLT(EltVT)) {
    3093             : 
    3094             :         // Convert shuffle node.
    3095             :         // If original node was v4i64 and the new EltVT is i32,
    3096             :         // cast operands to v8i32 and re-build the mask.
    3097             : 
    3098             :         // Calculate new VT, the size of the new VT should be equal to original.
    3099             :         EVT NewVT =
    3100           0 :             EVT::getVectorVT(*DAG.getContext(), NewEltVT,
    3101           0 :                              VT.getSizeInBits() / NewEltVT.getSizeInBits());
    3102             :         assert(NewVT.bitsEq(VT));
    3103             : 
    3104             :         // cast operands to new VT
    3105           0 :         Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
    3106           0 :         Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
    3107             : 
    3108             :         // Convert the shuffle mask
    3109             :         unsigned int factor =
    3110           0 :                          NewVT.getVectorNumElements()/VT.getVectorNumElements();
    3111             : 
    3112             :         // EltVT gets smaller
    3113             :         assert(factor > 0);
    3114             : 
    3115           0 :         for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
    3116           0 :           if (Mask[i] < 0) {
    3117           0 :             for (unsigned fi = 0; fi < factor; ++fi)
    3118           0 :               NewMask.push_back(Mask[i]);
    3119             :           }
    3120             :           else {
    3121           0 :             for (unsigned fi = 0; fi < factor; ++fi)
    3122           0 :               NewMask.push_back(Mask[i]*factor+fi);
    3123             :           }
    3124             :         }
    3125           0 :         Mask = NewMask;
    3126           0 :         VT = NewVT;
    3127             :       }
    3128          11 :       EltVT = NewEltVT;
    3129             :     }
    3130         181 :     unsigned NumElems = VT.getVectorNumElements();
    3131         362 :     SmallVector<SDValue, 16> Ops;
    3132        1305 :     for (unsigned i = 0; i != NumElems; ++i) {
    3133        2271 :       if (Mask[i] < 0) {
    3134          23 :         Ops.push_back(DAG.getUNDEF(EltVT));
    3135          23 :         continue;
    3136             :       }
    3137        1101 :       unsigned Idx = Mask[i];
    3138        1101 :       if (Idx < NumElems)
    3139        1190 :         Ops.push_back(DAG.getNode(
    3140             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
    3141        2380 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3142             :       else
    3143        1012 :         Ops.push_back(DAG.getNode(
    3144             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
    3145         506 :             DAG.getConstant(Idx - NumElems, dl,
    3146        2530 :                             TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3147             :     }
    3148             : 
    3149         362 :     Tmp1 = DAG.getBuildVector(VT, dl, Ops);
    3150             :     // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
    3151         543 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
    3152         181 :     Results.push_back(Tmp1);
    3153             :     break;
    3154             :   }
    3155         144 :   case ISD::EXTRACT_ELEMENT: {
    3156         432 :     EVT OpTy = Node->getOperand(0).getValueType();
    3157         576 :     if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
    3158             :       // 1 -> Hi
    3159         168 :       Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
    3160          56 :                          DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
    3161          56 :                                          TLI.getShiftAmountTy(
    3162         112 :                                              Node->getOperand(0).getValueType(),
    3163         280 :                                              DAG.getDataLayout())));
    3164         168 :       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
    3165             :     } else {
    3166             :       // 0 -> Lo
    3167         176 :       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
    3168         352 :                          Node->getOperand(0));
    3169             :     }
    3170         144 :     Results.push_back(Tmp1);
    3171             :     break;
    3172             :   }
    3173          98 :   case ISD::STACKSAVE:
    3174             :     // Expand to CopyFromReg if the target set
    3175             :     // StackPointerRegisterToSaveRestore.
    3176          98 :     if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
    3177         294 :       Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
    3178         196 :                                            Node->getValueType(0)));
    3179         196 :       Results.push_back(Results[0].getValue(1));
    3180             :     } else {
    3181           0 :       Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
    3182           0 :       Results.push_back(Node->getOperand(0));
    3183             :     }
    3184             :     break;
    3185          20 :   case ISD::STACKRESTORE:
    3186             :     // Expand to CopyToReg if the target set
    3187             :     // StackPointerRegisterToSaveRestore.
    3188          20 :     if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
    3189          60 :       Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
    3190          40 :                                          Node->getOperand(1)));
    3191             :     } else {
    3192           0 :       Results.push_back(Node->getOperand(0));
    3193             :     }
    3194             :     break;
    3195           0 :   case ISD::GET_DYNAMIC_AREA_OFFSET:
    3196           0 :     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
    3197           0 :     Results.push_back(Results[0].getValue(0));
    3198           0 :     break;
    3199          42 :   case ISD::FCOPYSIGN:
    3200          42 :     Results.push_back(ExpandFCOPYSIGN(Node));
    3201          42 :     break;
    3202           1 :   case ISD::FNEG:
    3203             :     // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
    3204           2 :     Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
    3205             :     // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
    3206           2 :     Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
    3207           4 :                        Node->getOperand(0));
    3208           1 :     Results.push_back(Tmp1);
    3209           1 :     break;
    3210           2 :   case ISD::FABS:
    3211           2 :     Results.push_back(ExpandFABS(Node));
    3212           2 :     break;
    3213           0 :   case ISD::SMIN:
    3214             :   case ISD::SMAX:
    3215             :   case ISD::UMIN:
    3216             :   case ISD::UMAX: {
    3217             :     // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
    3218             :     ISD::CondCode Pred;
    3219           0 :     switch (Node->getOpcode()) {
    3220           0 :     default: llvm_unreachable("How did we get here?");
    3221             :     case ISD::SMAX: Pred = ISD::SETGT; break;
    3222           0 :     case ISD::SMIN: Pred = ISD::SETLT; break;
    3223           0 :     case ISD::UMAX: Pred = ISD::SETUGT; break;
    3224           0 :     case ISD::UMIN: Pred = ISD::SETULT; break;
    3225             :     }
    3226           0 :     Tmp1 = Node->getOperand(0);
    3227           0 :     Tmp2 = Node->getOperand(1);
    3228           0 :     Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
    3229           0 :     Results.push_back(Tmp1);
    3230           0 :     break;
    3231             :   }
    3232             : 
    3233         281 :   case ISD::FSIN:
    3234             :   case ISD::FCOS: {
    3235         562 :     EVT VT = Node->getValueType(0);
    3236             :     // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
    3237             :     // fcos which share the same operand and both are used.
    3238         542 :     if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
    3239         522 :          isSinCosLibcallAvailable(Node, TLI))
    3240         139 :         && useSinCos(Node)) {
    3241          78 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3242         156 :       Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
    3243          78 :       if (Node->getOpcode() == ISD::FCOS)
    3244          78 :         Tmp1 = Tmp1.getValue(1);
    3245          78 :       Results.push_back(Tmp1);
    3246             :     }
    3247             :     break;
    3248             :   }
    3249           0 :   case ISD::FMAD:
    3250           0 :     llvm_unreachable("Illegal fmad should never be formed");
    3251             : 
    3252         157 :   case ISD::FP16_TO_FP:
    3253         314 :     if (Node->getValueType(0) != MVT::f32) {
    3254             :       // We can extend to types bigger than f32 in two steps without changing
    3255             :       // the result. Since "f16 -> f32" is much more commonly available, give
    3256             :       // CodeGen the option of emitting that before resorting to a libcall.
    3257             :       SDValue Res =
    3258           0 :           DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
    3259           0 :       Results.push_back(
    3260           0 :           DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
    3261             :     }
    3262             :     break;
    3263         345 :   case ISD::FP_TO_FP16:
    3264         345 :     if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
    3265          30 :       SDValue Op = Node->getOperand(0);
    3266          15 :       MVT SVT = Op.getSimpleValueType();
    3267          15 :       if ((SVT == MVT::f64 || SVT == MVT::f80) &&
    3268          30 :           TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
    3269             :         // Under fastmath, we can expand this node into a fround followed by
    3270             :         // a float-half conversion.
    3271          13 :         SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
    3272          26 :                                        DAG.getIntPtrConstant(0, dl));
    3273          13 :         Results.push_back(
    3274          52 :             DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
    3275             :       }
    3276             :     }
    3277             :     break;
    3278        5108 :   case ISD::ConstantFP: {
    3279        5108 :     ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
    3280             :     // Check to see if this FP immediate is already legal.
    3281             :     // If this is a legal constant, turn it into a TargetConstantFP node.
    3282       15324 :     if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
    3283        2786 :       Results.push_back(ExpandConstantFP(CFP, true));
    3284             :     break;
    3285             :   }
    3286           0 :   case ISD::Constant: {
    3287           0 :     ConstantSDNode *CP = cast<ConstantSDNode>(Node);
    3288           0 :     Results.push_back(ExpandConstant(CP));
    3289           0 :     break;
    3290             :   }
    3291         168 :   case ISD::FSUB: {
    3292         336 :     EVT VT = Node->getValueType(0);
    3293         168 :     if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
    3294         164 :         TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
    3295         328 :       const SDNodeFlags Flags = Node->getFlags();
    3296         492 :       Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
    3297         328 :       Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
    3298         164 :       Results.push_back(Tmp1);
    3299             :     }
    3300             :     break;
    3301             :   }
    3302           0 :   case ISD::SUB: {
    3303           0 :     EVT VT = Node->getValueType(0);
    3304             :     assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
    3305             :            TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
    3306             :            "Don't know how to expand this subtraction!");
    3307           0 :     Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
    3308           0 :                DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
    3309           0 :                                VT));
    3310           0 :     Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
    3311           0 :     Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
    3312             :     break;
    3313             :   }
    3314         910 :   case ISD::UREM:
    3315             :   case ISD::SREM: {
    3316        1820 :     EVT VT = Node->getValueType(0);
    3317         910 :     bool isSigned = Node->getOpcode() == ISD::SREM;
    3318         910 :     unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
    3319         910 :     unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3320        1820 :     Tmp2 = Node->getOperand(0);
    3321        1820 :     Tmp3 = Node->getOperand(1);
    3322         910 :     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
    3323         724 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3324        1448 :       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
    3325         724 :       Results.push_back(Tmp1);
    3326         186 :     } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
    3327             :       // X % Y -> X-X/Y*Y
    3328         312 :       Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
    3329         312 :       Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
    3330         312 :       Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
    3331         156 :       Results.push_back(Tmp1);
    3332             :     }
    3333             :     break;
    3334             :   }
    3335         657 :   case ISD::UDIV:
    3336             :   case ISD::SDIV: {
    3337         657 :     bool isSigned = Node->getOpcode() == ISD::SDIV;
    3338         657 :     unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3339        1314 :     EVT VT = Node->getValueType(0);
    3340         657 :     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
    3341         649 :       SDVTList VTs = DAG.getVTList(VT, VT);
    3342        1947 :       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
    3343        1298 :                          Node->getOperand(1));
    3344         649 :       Results.push_back(Tmp1);
    3345             :     }
    3346             :     break;
    3347             :   }
    3348         287 :   case ISD::MULHU:
    3349             :   case ISD::MULHS: {
    3350             :     unsigned ExpandOpcode =
    3351         287 :         Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
    3352         574 :     EVT VT = Node->getValueType(0);
    3353         287 :     SDVTList VTs = DAG.getVTList(VT, VT);
    3354             : 
    3355         861 :     Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
    3356         574 :                        Node->getOperand(1));
    3357         574 :     Results.push_back(Tmp1.getValue(1));
    3358             :     break;
    3359             :   }
    3360           0 :   case ISD::UMUL_LOHI:
    3361             :   case ISD::SMUL_LOHI: {
    3362           0 :     SDValue LHS = Node->getOperand(0);
    3363           0 :     SDValue RHS = Node->getOperand(1);
    3364           0 :     MVT VT = LHS.getSimpleValueType();
    3365             :     unsigned MULHOpcode =
    3366           0 :         Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
    3367             : 
    3368           0 :     if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
    3369           0 :       Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
    3370           0 :       Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
    3371           0 :       break;
    3372             :     }
    3373             : 
    3374           0 :     SmallVector<SDValue, 4> Halves;
    3375           0 :     EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
    3376             :     assert(TLI.isTypeLegal(HalfType));
    3377           0 :     if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
    3378             :                            HalfType, DAG,
    3379           0 :                            TargetLowering::MulExpansionKind::Always)) {
    3380           0 :       for (unsigned i = 0; i < 2; ++i) {
    3381           0 :         SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
    3382           0 :         SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
    3383           0 :         SDValue Shift = DAG.getConstant(
    3384           0 :             HalfType.getScalarSizeInBits(), dl,
    3385           0 :             TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
    3386           0 :         Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
    3387           0 :         Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
    3388             :       }
    3389             :       break;
    3390             :     }
    3391             :     break;
    3392             :   }
    3393          50 :   case ISD::MUL: {
    3394         100 :     EVT VT = Node->getValueType(0);
    3395          50 :     SDVTList VTs = DAG.getVTList(VT, VT);
    3396             :     // See if multiply or divide can be lowered using two-result operations.
    3397             :     // We just need the low half of the multiply; try both the signed
    3398             :     // and unsigned forms. If the target supports both SMUL_LOHI and
    3399             :     // UMUL_LOHI, form a preference by checking which forms of plain
    3400             :     // MULH it supports.
    3401         100 :     bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
    3402         100 :     bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
    3403         100 :     bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
    3404         100 :     bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
    3405          50 :     unsigned OpToUse = 0;
    3406          50 :     if (HasSMUL_LOHI && !HasMULHS) {
    3407             :       OpToUse = ISD::SMUL_LOHI;
    3408          43 :     } else if (HasUMUL_LOHI && !HasMULHU) {
    3409             :       OpToUse = ISD::UMUL_LOHI;
    3410          43 :     } else if (HasSMUL_LOHI) {
    3411             :       OpToUse = ISD::SMUL_LOHI;
    3412          43 :     } else if (HasUMUL_LOHI) {
    3413             :       OpToUse = ISD::UMUL_LOHI;
    3414             :     }
    3415             :     if (OpToUse) {
    3416          21 :       Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
    3417          14 :                                     Node->getOperand(1)));
    3418           7 :       break;
    3419             :     }
    3420             : 
    3421          43 :     SDValue Lo, Hi;
    3422          43 :     EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
    3423          43 :     if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
    3424          43 :         TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
    3425          43 :         TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
    3426          86 :         TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
    3427          49 :         TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
    3428          43 :                       TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
    3429          74 :       Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
    3430          74 :       Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
    3431             :       SDValue Shift =
    3432          74 :           DAG.getConstant(HalfType.getSizeInBits(), dl,
    3433         111 :                           TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
    3434          74 :       Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
    3435          74 :       Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
    3436             :     }
    3437             :     break;
    3438             :   }
    3439          25 :   case ISD::SADDO:
    3440             :   case ISD::SSUBO: {
    3441          50 :     SDValue LHS = Node->getOperand(0);
    3442          50 :     SDValue RHS = Node->getOperand(1);
    3443          50 :     SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
    3444             :                               ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
    3445          50 :                               LHS, RHS);
    3446          25 :     Results.push_back(Sum);
    3447          50 :     EVT ResultType = Node->getValueType(1);
    3448          50 :     EVT OType = getSetCCResultType(Node->getValueType(0));
    3449             : 
    3450          50 :     SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
    3451             : 
    3452             :     //   LHSSign -> LHS >= 0
    3453             :     //   RHSSign -> RHS >= 0
    3454             :     //   SumSign -> Sum >= 0
    3455             :     //
    3456             :     //   Add:
    3457             :     //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
    3458             :     //   Sub:
    3459             :     //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
    3460             :     //
    3461          25 :     SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
    3462          25 :     SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
    3463          25 :     SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
    3464          25 :                                       Node->getOpcode() == ISD::SADDO ?
    3465          25 :                                       ISD::SETEQ : ISD::SETNE);
    3466             : 
    3467          25 :     SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
    3468          25 :     SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
    3469             : 
    3470          50 :     SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
    3471          25 :     Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
    3472             :     break;
    3473             :   }
    3474          14 :   case ISD::UADDO:
    3475             :   case ISD::USUBO: {
    3476          28 :     SDValue LHS = Node->getOperand(0);
    3477          28 :     SDValue RHS = Node->getOperand(1);
    3478          28 :     SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
    3479             :                               ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
    3480          28 :                               LHS, RHS);
    3481          14 :     Results.push_back(Sum);
    3482             : 
    3483          28 :     EVT ResultType = Node->getValueType(1);
    3484          28 :     EVT SetCCType = getSetCCResultType(Node->getValueType(0));
    3485             :     ISD::CondCode CC
    3486          14 :       = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
    3487          14 :     SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
    3488             : 
    3489          14 :     Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
    3490             :     break;
    3491             :   }
    3492          10 :   case ISD::UMULO:
    3493             :   case ISD::SMULO: {
    3494          20 :     EVT VT = Node->getValueType(0);
    3495          10 :     EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
    3496          20 :     SDValue LHS = Node->getOperand(0);
    3497          20 :     SDValue RHS = Node->getOperand(1);
    3498          10 :     SDValue BottomHalf;
    3499          10 :     SDValue TopHalf;
    3500             :     static const unsigned Ops[2][3] =
    3501             :         { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
    3502             :           { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
    3503          20 :     bool isSigned = Node->getOpcode() == ISD::SMULO;
    3504          10 :     if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
    3505          10 :       BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
    3506          10 :       TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
    3507           5 :     } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
    3508           0 :       BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
    3509           0 :                                RHS);
    3510           0 :       TopHalf = BottomHalf.getValue(1);
    3511           5 :     } else if (TLI.isTypeLegal(WideVT)) {
    3512           0 :       LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
    3513           0 :       RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
    3514           0 :       Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
    3515           0 :       BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
    3516           0 :                                DAG.getIntPtrConstant(0, dl));
    3517           0 :       TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
    3518           0 :                             DAG.getIntPtrConstant(1, dl));
    3519             :     } else {
    3520             :       // We can fall back to a libcall with an illegal type for the MUL if we
    3521             :       // have a libcall big enough.
    3522             :       // Also, we can fall back to a division in some cases, but that's a big
    3523             :       // performance hit in the general case.
    3524           5 :       RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    3525          10 :       if (WideVT == MVT::i16)
    3526             :         LC = RTLIB::MUL_I16;
    3527           9 :       else if (WideVT == MVT::i32)
    3528             :         LC = RTLIB::MUL_I32;
    3529           4 :       else if (WideVT == MVT::i64)
    3530             :         LC = RTLIB::MUL_I64;
    3531           0 :       else if (WideVT == MVT::i128)
    3532           0 :         LC = RTLIB::MUL_I128;
    3533             :       assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
    3534             : 
    3535           5 :       SDValue HiLHS;
    3536           5 :       SDValue HiRHS;
    3537           5 :       if (isSigned) {
    3538             :         // The high part is obtained by SRA'ing all but one of the bits of low
    3539             :         // part.
    3540           1 :         unsigned LoSize = VT.getSizeInBits();
    3541           1 :         HiLHS =
    3542           3 :             DAG.getNode(ISD::SRA, dl, VT, LHS,
    3543           1 :                         DAG.getConstant(LoSize - 1, dl,
    3544           6 :                                         TLI.getPointerTy(DAG.getDataLayout())));
    3545           1 :         HiRHS =
    3546           3 :             DAG.getNode(ISD::SRA, dl, VT, RHS,
    3547             :                         DAG.getConstant(LoSize - 1, dl,
    3548           5 :                                         TLI.getPointerTy(DAG.getDataLayout())));
    3549             :       } else {
    3550           4 :           HiLHS = DAG.getConstant(0, dl, VT);
    3551           4 :           HiRHS = DAG.getConstant(0, dl, VT);
    3552             :       }
    3553             : 
    3554             :       // Here we're passing the 2 arguments explicitly as 4 arguments that are
    3555             :       // pre-lowered to the correct types. This all depends upon WideVT not
    3556             :       // being a legal type for the architecture and thus has to be split to
    3557             :       // two arguments.
    3558           5 :       SDValue Ret;
    3559          10 :       if(DAG.getDataLayout().isLittleEndian()) {
    3560             :         // Halves of WideVT are packed into registers in different order
    3561             :         // depending on platform endianness. This is usually handled by
    3562             :         // the C calling convention, but we can't defer to it in
    3563             :         // the legalizer.
    3564           5 :         SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
    3565           5 :         Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
    3566             :       } else {
    3567           0 :         SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
    3568           0 :         Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
    3569             :       }
    3570             :       assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
    3571             :              "Ret value is a collection of constituent nodes holding result.");
    3572          10 :       BottomHalf = Ret.getOperand(0);
    3573          10 :       TopHalf = Ret.getOperand(1);
    3574             :     }
    3575             : 
    3576          10 :     if (isSigned) {
    3577          10 :       Tmp1 = DAG.getConstant(
    3578           5 :           VT.getSizeInBits() - 1, dl,
    3579          20 :           TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
    3580          10 :       Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
    3581          10 :       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
    3582           5 :                              ISD::SETNE);
    3583             :     } else {
    3584          10 :       TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
    3585           5 :                              DAG.getConstant(0, dl, VT), ISD::SETNE);
    3586             :     }
    3587             : 
    3588             :     // Truncate the result if SetCC returns a larger type than needed.
    3589          20 :     EVT RType = Node->getValueType(1);
    3590          10 :     if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
    3591           2 :       TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
    3592             : 
    3593             :     assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
    3594             :            "Unexpected result type for S/UMULO legalization");
    3595             : 
    3596          10 :     Results.push_back(BottomHalf);
    3597          10 :     Results.push_back(TopHalf);
    3598             :     break;
    3599             :   }
    3600          10 :   case ISD::BUILD_PAIR: {
    3601          20 :     EVT PairTy = Node->getValueType(0);
    3602          30 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
    3603          30 :     Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
    3604          20 :     Tmp2 = DAG.getNode(
    3605             :         ISD::SHL, dl, PairTy, Tmp2,
    3606          10 :         DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
    3607          40 :                         TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
    3608          20 :     Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
    3609             :     break;
    3610             :   }
    3611         188 :   case ISD::SELECT:
    3612         376 :     Tmp1 = Node->getOperand(0);
    3613         376 :     Tmp2 = Node->getOperand(1);
    3614         376 :     Tmp3 = Node->getOperand(2);
    3615         376 :     if (Tmp1.getOpcode() == ISD::SETCC) {
    3616         556 :       Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
    3617             :                              Tmp2, Tmp3,
    3618         417 :                              cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
    3619             :     } else {
    3620          98 :       Tmp1 = DAG.getSelectCC(dl, Tmp1,
    3621             :                              DAG.getConstant(0, dl, Tmp1.getValueType()),
    3622          98 :                              Tmp2, Tmp3, ISD::SETNE);
    3623             :     }
    3624         188 :     Results.push_back(Tmp1);
    3625         188 :     break;
    3626         153 :   case ISD::BR_JT: {
    3627         306 :     SDValue Chain = Node->getOperand(0);
    3628         306 :     SDValue Table = Node->getOperand(1);
    3629         306 :     SDValue Index = Node->getOperand(2);
    3630             : 
    3631         306 :     const DataLayout &TD = DAG.getDataLayout();
    3632         459 :     EVT PTy = TLI.getPointerTy(TD);
    3633             : 
    3634             :     unsigned EntrySize =
    3635         153 :       DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
    3636             : 
    3637         306 :     Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
    3638         459 :                         DAG.getConstant(EntrySize, dl, Index.getValueType()));
    3639         153 :     SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
    3640         306 :                                Index, Table);
    3641             : 
    3642         153 :     EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
    3643         153 :     SDValue LD = DAG.getExtLoad(
    3644             :         ISD::SEXTLOAD, dl, PTy, Chain, Addr,
    3645         153 :         MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
    3646         153 :     Addr = LD;
    3647         153 :     if (TLI.isJumpTableRelative()) {
    3648             :       // For PIC, the sequence is:
    3649             :       // BRIND(load(Jumptable + index) + RelocBase)
    3650             :       // RelocBase can be JumpTable, GOT or some sort of global base.
    3651         104 :       Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
    3652         104 :                           TLI.getPICJumpTableRelocBase(Table, DAG));
    3653             :     }
    3654         612 :     Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
    3655         153 :     Results.push_back(Tmp1);
    3656             :     break;
    3657             :   }
    3658         389 :   case ISD::BRCOND:
    3659             :     // Expand brcond's setcc into its constituent parts and create a BR_CC
    3660             :     // Node.
    3661         778 :     Tmp1 = Node->getOperand(0);
    3662         778 :     Tmp2 = Node->getOperand(1);
    3663         778 :     if (Tmp2.getOpcode() == ISD::SETCC) {
    3664           4 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
    3665           4 :                          Tmp1, Tmp2.getOperand(2),
    3666           6 :                          Tmp2.getOperand(0), Tmp2.getOperand(1),
    3667           8 :                          Node->getOperand(2));
    3668             :     } else {
    3669             :       // We test only the i1 bit.  Skip the AND if UNDEF or another AND.
    3670        1161 :       if (Tmp2.isUndef() ||
    3671         387 :           (Tmp2.getOpcode() == ISD::AND &&
    3672         208 :            isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
    3673         198 :            dyn_cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
    3674          66 :         Tmp3 = Tmp2;
    3675             :       else
    3676         642 :         Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
    3677         963 :                            DAG.getConstant(1, dl, Tmp2.getValueType()));
    3678         774 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
    3679         387 :                          DAG.getCondCode(ISD::SETNE), Tmp3,
    3680             :                          DAG.getConstant(0, dl, Tmp3.getValueType()),
    3681        2322 :                          Node->getOperand(2));
    3682             :     }
    3683         389 :     Results.push_back(Tmp1);
    3684         389 :     break;
    3685        2171 :   case ISD::SETCC: {
    3686        4342 :     Tmp1 = Node->getOperand(0);
    3687        4342 :     Tmp2 = Node->getOperand(1);
    3688        4342 :     Tmp3 = Node->getOperand(2);
    3689        4342 :     bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
    3690        2171 :                                            Tmp3, NeedInvert, dl);
    3691             : 
    3692        2171 :     if (Legalized) {
    3693             :       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
    3694             :       // condition code, create a new SETCC node.
    3695         325 :       if (Tmp3.getNode())
    3696         472 :         Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
    3697         472 :                            Tmp1, Tmp2, Tmp3);
    3698             : 
    3699             :       // If we expanded the SETCC by inverting the condition code, then wrap
    3700             :       // the existing SETCC in a NOT to restore the intended condition.
    3701         325 :       if (NeedInvert)
    3702          32 :         Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
    3703             : 
    3704         325 :       Results.push_back(Tmp1);
    3705         325 :       break;
    3706             :     }
    3707             : 
    3708             :     // Otherwise, SETCC for the given comparison type must be completely
    3709             :     // illegal; expand it into a SELECT_CC.
    3710        3692 :     EVT VT = Node->getValueType(0);
    3711             :     int TrueValue;
    3712        3692 :     switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
    3713         480 :     case TargetLowering::ZeroOrOneBooleanContent:
    3714             :     case TargetLowering::UndefinedBooleanContent:
    3715         480 :       TrueValue = 1;
    3716         480 :       break;
    3717        1366 :     case TargetLowering::ZeroOrNegativeOneBooleanContent:
    3718        1366 :       TrueValue = -1;
    3719        1366 :       break;
    3720             :     }
    3721        3692 :     Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
    3722        1846 :                        DAG.getConstant(TrueValue, dl, VT),
    3723             :                        DAG.getConstant(0, dl, VT),
    3724        3692 :                        Tmp3);
    3725        1846 :     Results.push_back(Tmp1);
    3726        1846 :     break;
    3727             :   }
    3728        3316 :   case ISD::SELECT_CC: {
    3729        6632 :     Tmp1 = Node->getOperand(0);   // LHS
    3730        6632 :     Tmp2 = Node->getOperand(1);   // RHS
    3731        6632 :     Tmp3 = Node->getOperand(2);   // True
    3732        6632 :     Tmp4 = Node->getOperand(3);   // False
    3733        6632 :     EVT VT = Node->getValueType(0);
    3734        6632 :     SDValue CC = Node->getOperand(4);
    3735        3316 :     ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
    3736             : 
    3737        6933 :     if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
    3738             :       // If the condition code is legal, then we need to expand this
    3739             :       // node using SETCC and SELECT.
    3740        6030 :       EVT CmpVT = Tmp1.getValueType();
    3741             :       assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
    3742             :              "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
    3743             :              "expanded.");
    3744             :       EVT CCVT =
    3745        6030 :           TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
    3746        3015 :       SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
    3747        3015 :       Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
    3748             :       break;
    3749             :     }
    3750             : 
    3751             :     // SELECT_CC is legal, so the condition code must not be.
    3752         301 :     bool Legalized = false;
    3753             :     // Try to legalize by inverting the condition.  This is for targets that
    3754             :     // might support an ordered version of a condition, but not the unordered
    3755             :     // version (or vice versa).
    3756         301 :     ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
    3757         903 :                                                Tmp1.getValueType().isInteger());
    3758         647 :     if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
    3759             :       // Use the new condition code and swap true and false
    3760         256 :       Legalized = true;
    3761         256 :       Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
    3762             :     } else {
    3763             :       // If The inverse is not legal, then try to swap the arguments using
    3764             :       // the inverse condition code.
    3765          45 :       ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
    3766         123 :       if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
    3767             :         // The swapped inverse condition is legal, so swap true and false,
    3768             :         // lhs and rhs.
    3769          12 :         Legalized = true;
    3770          12 :         Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
    3771             :       }
    3772             :     }
    3773             : 
    3774             :     if (!Legalized) {
    3775          66 :       Legalized = LegalizeSetCCCondCode(
    3776             :           getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
    3777             :           dl);
    3778             : 
    3779             :       assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
    3780             : 
    3781             :       // If we expanded the SETCC by inverting the condition code, then swap
    3782             :       // the True/False operands to match.
    3783          33 :       if (NeedInvert)
    3784             :         std::swap(Tmp3, Tmp4);
    3785             : 
    3786             :       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
    3787             :       // condition code, create a new SELECT_CC node.
    3788          33 :       if (CC.getNode()) {
    3789          26 :         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
    3790          26 :                            Tmp1, Tmp2, Tmp3, Tmp4, CC);
    3791             :       } else {
    3792          40 :         Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
    3793          20 :         CC = DAG.getCondCode(ISD::SETNE);
    3794          40 :         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
    3795          40 :                            Tmp2, Tmp3, Tmp4, CC);
    3796             :       }
    3797             :     }
    3798         301 :     Results.push_back(Tmp1);
    3799         301 :     break;
    3800             :   }
    3801           9 :   case ISD::BR_CC: {
    3802          18 :     Tmp1 = Node->getOperand(0);              // Chain
    3803          18 :     Tmp2 = Node->getOperand(2);              // LHS
    3804          18 :     Tmp3 = Node->getOperand(3);              // RHS
    3805          18 :     Tmp4 = Node->getOperand(1);              // CC
    3806             : 
    3807          18 :     bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
    3808           9 :         Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
    3809             :     (void)Legalized;
    3810             :     assert(Legalized && "Can't legalize BR_CC with legal condition!");
    3811             : 
    3812             :     // If we expanded the SETCC by inverting the condition code, then wrap
    3813             :     // the existing SETCC in a NOT to restore the intended condition.
    3814           9 :     if (NeedInvert)
    3815           0 :       Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
    3816             : 
    3817             :     // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
    3818             :     // node.
    3819           9 :     if (Tmp4.getNode()) {
    3820           0 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
    3821           0 :                          Tmp4, Tmp2, Tmp3, Node->getOperand(4));
    3822             :     } else {
    3823          18 :       Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
    3824           9 :       Tmp4 = DAG.getCondCode(ISD::SETNE);
    3825          18 :       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
    3826          27 :                          Tmp2, Tmp3, Node->getOperand(4));
    3827             :     }
    3828           9 :     Results.push_back(Tmp1);
    3829           9 :     break;
    3830             :   }
    3831       17304 :   case ISD::BUILD_VECTOR:
    3832       17304 :     Results.push_back(ExpandBUILD_VECTOR(Node));
    3833       17304 :     break;
    3834          10 :   case ISD::SRA:
    3835             :   case ISD::SRL:
    3836             :   case ISD::SHL: {
    3837             :     // Scalarize vector SRA/SRL/SHL.
    3838          20 :     EVT VT = Node->getValueType(0);
    3839             :     assert(VT.isVector() && "Unable to legalize non-vector shift");
    3840             :     assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
    3841          10 :     unsigned NumElem = VT.getVectorNumElements();
    3842             : 
    3843          20 :     SmallVector<SDValue, 8> Scalars;
    3844          30 :     for (unsigned Idx = 0; Idx < NumElem; Idx++) {
    3845          20 :       SDValue Ex = DAG.getNode(
    3846          40 :           ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
    3847          80 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3848          20 :       SDValue Sh = DAG.getNode(
    3849          40 :           ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
    3850          80 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3851          40 :       Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
    3852          40 :                                     VT.getScalarType(), Ex, Sh));
    3853             :     }
    3854             : 
    3855          30 :     SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
    3856          10 :     ReplaceNode(SDValue(Node, 0), Result);
    3857             :     break;
    3858             :   }
    3859             :   case ISD::GLOBAL_OFFSET_TABLE:
    3860             :   case ISD::GlobalAddress:
    3861             :   case ISD::GlobalTLSAddress:
    3862             :   case ISD::ExternalSymbol:
    3863             :   case ISD::ConstantPool:
    3864             :   case ISD::JumpTable:
    3865             :   case ISD::INTRINSIC_W_CHAIN:
    3866             :   case ISD::INTRINSIC_WO_CHAIN:
    3867             :   case ISD::INTRINSIC_VOID:
    3868             :     // FIXME: Custom lowering for these operations shouldn't return null!
    3869             :     break;
    3870             :   }
    3871             : 
    3872             :   // Replace the original node with the legalized result.
    3873      228092 :   if (Results.empty())
    3874             :     return false;
    3875             : 
    3876      357744 :   ReplaceNode(Node, Results.data());
    3877      178872 :   return true;
    3878             : }
    3879             : 
    3880       49292 : void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
    3881       98584 :   SmallVector<SDValue, 8> Results;
    3882       98584 :   SDLoc dl(Node);
    3883       49292 :   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
    3884       98584 :   unsigned Opc = Node->getOpcode();
    3885       49292 :   switch (Opc) {
    3886          13 :   case ISD::ATOMIC_FENCE: {
    3887             :     // If the target didn't lower this, lower it to '__sync_synchronize()' call
    3888             :     // FIXME: handle "fence singlethread" more efficiently.
    3889          26 :     TargetLowering::ArgListTy Args;
    3890             : 
    3891          26 :     TargetLowering::CallLoweringInfo CLI(DAG);
    3892          13 :     CLI.setDebugLoc(dl)
    3893          39 :         .setChain(Node->getOperand(0))
    3894          13 :         .setLibCallee(
    3895          13 :             CallingConv::C, Type::getVoidTy(*DAG.getContext()),
    3896          13 :             DAG.getExternalSymbol("__sync_synchronize",
    3897          13 :                                   TLI.getPointerTy(DAG.getDataLayout())),
    3898          91 :             std::move(Args));
    3899             : 
    3900          13 :     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
    3901             : 
    3902          13 :     Results.push_back(CallResult.second);
    3903             :     break;
    3904             :   }
    3905             :   // By default, atomic intrinsics are marked Legal and lowered. Targets
    3906             :   // which don't support them directly, however, may want libcalls, in which
    3907             :   // case they mark them Expand, and we get here.
    3908         309 :   case ISD::ATOMIC_SWAP:
    3909             :   case ISD::ATOMIC_LOAD_ADD:
    3910             :   case ISD::ATOMIC_LOAD_SUB:
    3911             :   case ISD::ATOMIC_LOAD_AND:
    3912             :   case ISD::ATOMIC_LOAD_OR:
    3913             :   case ISD::ATOMIC_LOAD_XOR:
    3914             :   case ISD::ATOMIC_LOAD_NAND:
    3915             :   case ISD::ATOMIC_LOAD_MIN:
    3916             :   case ISD::ATOMIC_LOAD_MAX:
    3917             :   case ISD::ATOMIC_LOAD_UMIN:
    3918             :   case ISD::ATOMIC_LOAD_UMAX:
    3919             :   case ISD::ATOMIC_CMP_SWAP: {
    3920         309 :     MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
    3921         309 :     RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
    3922             :     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
    3923             : 
    3924         309 :     std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
    3925         309 :     Results.push_back(Tmp.first);
    3926         309 :     Results.push_back(Tmp.second);
    3927             :     break;
    3928             :   }
    3929           2 :   case ISD::TRAP: {
    3930             :     // If this operation is not supported, lower it to 'abort()' call
    3931           4 :     TargetLowering::ArgListTy Args;
    3932           4 :     TargetLowering::CallLoweringInfo CLI(DAG);
    3933           2 :     CLI.setDebugLoc(dl)
    3934           6 :         .setChain(Node->getOperand(0))
    3935           4 :         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
    3936           2 :                       DAG.getExternalSymbol(
    3937           2 :                           "abort", TLI.getPointerTy(DAG.getDataLayout())),
    3938          14 :                       std::move(Args));
    3939           2 :     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
    3940             : 
    3941           2 :     Results.push_back(CallResult.second);
    3942             :     break;
    3943             :   }
    3944          45 :   case ISD::FMINNUM:
    3945          45 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
    3946             :                                       RTLIB::FMIN_F80, RTLIB::FMIN_F128,
    3947             :                                       RTLIB::FMIN_PPCF128));
    3948          45 :     break;
    3949          47 :   case ISD::FMAXNUM:
    3950          47 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
    3951             :                                       RTLIB::FMAX_F80, RTLIB::FMAX_F128,
    3952             :                                       RTLIB::FMAX_PPCF128));
    3953          47 :     break;
    3954           4 :   case ISD::FSQRT:
    3955             :   case ISD::STRICT_FSQRT:
    3956           4 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
    3957             :                                       RTLIB::SQRT_F80, RTLIB::SQRT_F128,
    3958             :                                       RTLIB::SQRT_PPCF128));
    3959           4 :     break;
    3960         115 :   case ISD::FSIN:
    3961             :   case ISD::STRICT_FSIN:
    3962         115 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
    3963             :                                       RTLIB::SIN_F80, RTLIB::SIN_F128,
    3964             :                                       RTLIB::SIN_PPCF128));
    3965         115 :     break;
    3966          92 :   case ISD::FCOS:
    3967             :   case ISD::STRICT_FCOS:
    3968          92 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
    3969             :                                       RTLIB::COS_F80, RTLIB::COS_F128,
    3970             :                                       RTLIB::COS_PPCF128));
    3971          92 :     break;
    3972          30 :   case ISD::FSINCOS:
    3973             :     // Expand into sincos libcall.
    3974          30 :     ExpandSinCosLibCall(Node, Results);
    3975          30 :     break;
    3976          56 :   case ISD::FLOG:
    3977             :   case ISD::STRICT_FLOG:
    3978          56 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
    3979             :                                       RTLIB::LOG_F80, RTLIB::LOG_F128,
    3980             :                                       RTLIB::LOG_PPCF128));
    3981          56 :     break;
    3982          59 :   case ISD::FLOG2:
    3983             :   case ISD::STRICT_FLOG2:
    3984          59 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
    3985             :                                       RTLIB::LOG2_F80, RTLIB::LOG2_F128,
    3986             :                                       RTLIB::LOG2_PPCF128));
    3987          59 :     break;
    3988          60 :   case ISD::FLOG10:
    3989             :   case ISD::STRICT_FLOG10:
    3990          60 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
    3991             :                                       RTLIB::LOG10_F80, RTLIB::LOG10_F128,
    3992             :                                       RTLIB::LOG10_PPCF128));
    3993          60 :     break;
    3994          72 :   case ISD::FEXP:
    3995             :   case ISD::STRICT_FEXP:
    3996          72 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
    3997             :                                       RTLIB::EXP_F80, RTLIB::EXP_F128,
    3998             :                                       RTLIB::EXP_PPCF128));
    3999          72 :     break;
    4000          59 :   case ISD::FEXP2:
    4001             :   case ISD::STRICT_FEXP2:
    4002          59 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
    4003             :                                       RTLIB::EXP2_F80, RTLIB::EXP2_F128,
    4004             :                                       RTLIB::EXP2_PPCF128));
    4005          59 :     break;
    4006          30 :   case ISD::FTRUNC:
    4007          30 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
    4008             :                                       RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
    4009             :                                       RTLIB::TRUNC_PPCF128));
    4010          30 :     break;
    4011          38 :   case ISD::FFLOOR:
    4012          38 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
    4013             :                                       RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
    4014             :                                       RTLIB::FLOOR_PPCF128));
    4015          38 :     break;
    4016          20 :   case ISD::FCEIL:
    4017          20 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
    4018             :                                       RTLIB::CEIL_F80, RTLIB::CEIL_F128,
    4019             :                                       RTLIB::CEIL_PPCF128));
    4020          20 :     break;
    4021          25 :   case ISD::FRINT:
    4022             :   case ISD::STRICT_FRINT:
    4023          25 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
    4024             :                                       RTLIB::RINT_F80, RTLIB::RINT_F128,
    4025             :                                       RTLIB::RINT_PPCF128));
    4026          25 :     break;
    4027          48 :   case ISD::FNEARBYINT:
    4028             :   case ISD::STRICT_FNEARBYINT:
    4029          48 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
    4030             :                                       RTLIB::NEARBYINT_F64,
    4031             :                                       RTLIB::NEARBYINT_F80,
    4032             :                                       RTLIB::NEARBYINT_F128,
    4033             :                                       RTLIB::NEARBYINT_PPCF128));
    4034          48 :     break;
    4035          17 :   case ISD::FROUND:
    4036          17 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
    4037             :                                       RTLIB::ROUND_F64,
    4038             :                                       RTLIB::ROUND_F80,
    4039             :                                       RTLIB::ROUND_F128,
    4040             :                                       RTLIB::ROUND_PPCF128));
    4041          17 :     break;
    4042          99 :   case ISD::FPOWI:
    4043             :   case ISD::STRICT_FPOWI:
    4044          99 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
    4045             :                                       RTLIB::POWI_F80, RTLIB::POWI_F128,
    4046             :                                       RTLIB::POWI_PPCF128));
    4047          99 :     break;
    4048          89 :   case ISD::FPOW:
    4049             :   case ISD::STRICT_FPOW:
    4050          89 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
    4051             :                                       RTLIB::POW_F80, RTLIB::POW_F128,
    4052             :                                       RTLIB::POW_PPCF128));
    4053          89 :     break;
    4054           7 :   case ISD::FDIV:
    4055           7 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
    4056             :                                       RTLIB::DIV_F80, RTLIB::DIV_F128,
    4057             :                                       RTLIB::DIV_PPCF128));
    4058           7 :     break;
    4059          85 :   case ISD::FREM:
    4060          85 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
    4061             :                                       RTLIB::REM_F80, RTLIB::REM_F128,
    4062             :                                       RTLIB::REM_PPCF128));
    4063          85 :     break;
    4064         160 :   case ISD::FMA:
    4065             :   case ISD::STRICT_FMA:
    4066         160 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
    4067             :                                       RTLIB::FMA_F80, RTLIB::FMA_F128,
    4068             :                                       RTLIB::FMA_PPCF128));
    4069         160 :     break;
    4070          12 :   case ISD::FADD:
    4071          12 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
    4072             :                                       RTLIB::ADD_F80, RTLIB::ADD_F128,
    4073             :                                       RTLIB::ADD_PPCF128));
    4074          12 :     break;
    4075           7 :   case ISD::FMUL:
    4076           7 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
    4077             :                                       RTLIB::MUL_F80, RTLIB::MUL_F128,
    4078             :                                       RTLIB::MUL_PPCF128));
    4079           7 :     break;
    4080         157 :   case ISD::FP16_TO_FP:
    4081         314 :     if (Node->getValueType(0) == MVT::f32) {
    4082         157 :       Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
    4083             :     }
    4084             :     break;
    4085         332 :   case ISD::FP_TO_FP16: {
    4086             :     RTLIB::Libcall LC =
    4087         996 :         RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
    4088             :     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
    4089         332 :     Results.push_back(ExpandLibCall(LC, Node, false));
    4090         332 :     break;
    4091             :   }
    4092           4 :   case ISD::FSUB:
    4093           4 :     Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
    4094             :                                       RTLIB::SUB_F80, RTLIB::SUB_F128,
    4095             :                                       RTLIB::SUB_PPCF128));
    4096           4 :     break;
    4097          19 :   case ISD::SREM:
    4098          19 :     Results.push_back(ExpandIntLibCall(Node, true,
    4099             :                                        RTLIB::SREM_I8,
    4100             :                                        RTLIB::SREM_I16, RTLIB::SREM_I32,
    4101             :                                        RTLIB::SREM_I64, RTLIB::SREM_I128));
    4102          19 :     break;
    4103          13 :   case ISD::UREM:
    4104          13 :     Results.push_back(ExpandIntLibCall(Node, false,
    4105             :                                        RTLIB::UREM_I8,
    4106             :                                        RTLIB::UREM_I16, RTLIB::UREM_I32,
    4107             :                                        RTLIB::UREM_I64, RTLIB::UREM_I128));
    4108          13 :     break;
    4109          48 :   case ISD::SDIV:
    4110          48 :     Results.push_back(ExpandIntLibCall(Node, true,
    4111             :                                        RTLIB::SDIV_I8,
    4112             :                                        RTLIB::SDIV_I16, RTLIB::SDIV_I32,
    4113             :                                        RTLIB::SDIV_I64, RTLIB::SDIV_I128));
    4114          48 :     break;
    4115          20 :   case ISD::UDIV:
    4116          20 :     Results.push_back(ExpandIntLibCall(Node, false,
    4117             :                                        RTLIB::UDIV_I8,
    4118             :                                        RTLIB::UDIV_I16, RTLIB::UDIV_I32,
    4119             :                                        RTLIB::UDIV_I64, RTLIB::UDIV_I128));
    4120          20 :     break;
    4121          14 :   case ISD::SDIVREM:
    4122             :   case ISD::UDIVREM:
    4123             :     // Expand into divrem libcall
    4124          14 :     ExpandDivRemLibCall(Node, Results);
    4125          14 :     break;
    4126          16 :   case ISD::MUL:
    4127          16 :     Results.push_back(ExpandIntLibCall(Node, false,
    4128             :                                        RTLIB::MUL_I8,
    4129             :                                        RTLIB::MUL_I16, RTLIB::MUL_I32,
    4130             :                                        RTLIB::MUL_I64, RTLIB::MUL_I128));
    4131          16 :     break;
    4132             :   }
    4133             : 
    4134             :   // Replace the original node with the legalized result.
    4135       49292 :   if (!Results.empty())
    4136        4446 :     ReplaceNode(Node, Results.data());
    4137       49292 : }
    4138             : 
    4139             : // Determine the vector type to use in place of an original scalar element when
    4140             : // promoting equally sized vectors.
    4141        2805 : static MVT getPromotedVectorElementType(const TargetLowering &TLI,
    4142             :                                         MVT EltVT, MVT NewEltVT) {
    4143        2805 :   unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
    4144        2805 :   MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
    4145             :   assert(TLI.isTypeLegal(MidVT) && "unexpected");
    4146        2805 :   return MidVT;
    4147             : }
    4148             : 
    4149        6124 : void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
    4150       12248 :   SmallVector<SDValue, 8> Results;
    4151        6124 :   MVT OVT = Node->getSimpleValueType(0);
    4152       12200 :   if (Node->getOpcode() == ISD::UINT_TO_FP ||
    4153       12130 :       Node->getOpcode() == ISD::SINT_TO_FP ||
    4154       11931 :       Node->getOpcode() == ISD::SETCC ||
    4155       15468 :       Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
    4156        3467 :       Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
    4157        7983 :     OVT = Node->getOperand(0).getSimpleValueType();
    4158             :   }
    4159       12248 :   if (Node->getOpcode() == ISD::BR_CC)
    4160           3 :     OVT = Node->getOperand(2).getSimpleValueType();
    4161        6124 :   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
    4162       12248 :   SDLoc dl(Node);
    4163        6124 :   SDValue Tmp1, Tmp2, Tmp3;
    4164       12248 :   switch (Node->getOpcode()) {
    4165          33 :   case ISD::CTTZ:
    4166             :   case ISD::CTTZ_ZERO_UNDEF:
    4167             :   case ISD::CTLZ:
    4168             :   case ISD::CTLZ_ZERO_UNDEF:
    4169             :   case ISD::CTPOP:
    4170             :     // Zero extend the argument.
    4171         132 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
    4172          33 :     if (Node->getOpcode() == ISD::CTTZ) {
    4173             :       // The count is the same in the promoted type except if the original
    4174             :       // value was zero.  This can be handled by setting the bit just off
    4175             :       // the top of the original type.
    4176             :       auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
    4177           8 :                                         OVT.getSizeInBits());
    4178           8 :       Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
    4179          16 :                          DAG.getConstant(TopBit, dl, NVT));
    4180             :     }
    4181             :     // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
    4182             :     // already the correct result.
    4183         132 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4184          94 :     if (Node->getOpcode() == ISD::CTLZ ||
    4185          28 :         Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
    4186             :       // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
    4187          22 :       Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
    4188          33 :                           DAG.getConstant(NVT.getSizeInBits() -
    4189          66 :                                           OVT.getSizeInBits(), dl, NVT));
    4190             :     }
    4191          99 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4192          33 :     break;
    4193           0 :   case ISD::BITREVERSE:
    4194             :   case ISD::BSWAP: {
    4195           0 :     unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
    4196           0 :     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
    4197           0 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4198           0 :     Tmp1 = DAG.getNode(
    4199             :         ISD::SRL, dl, NVT, Tmp1,
    4200             :         DAG.getConstant(DiffBits, dl,
    4201           0 :                         TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
    4202           0 :     Results.push_back(Tmp1);
    4203           0 :     break;
    4204             :   }
    4205         108 :   case ISD::FP_TO_UINT:
    4206             :   case ISD::FP_TO_SINT:
    4207         216 :     Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
    4208         216 :                                  Node->getOpcode() == ISD::FP_TO_SINT, dl);
    4209         108 :     Results.push_back(Tmp1);
    4210         108 :     break;
    4211          70 :   case ISD::UINT_TO_FP:
    4212             :   case ISD::SINT_TO_FP:
    4213         140 :     Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
    4214         140 :                                  Node->getOpcode() == ISD::SINT_TO_FP, dl);
    4215          70 :     Results.push_back(Tmp1);
    4216          70 :     break;
    4217           1 :   case ISD::VAARG: {
    4218           2 :     SDValue Chain = Node->getOperand(0); // Get the chain.
    4219           2 :     SDValue Ptr = Node->getOperand(1); // Get the pointer.
    4220             : 
    4221             :     unsigned TruncOp;
    4222           1 :     if (OVT.isVector()) {
    4223             :       TruncOp = ISD::BITCAST;
    4224             :     } else {
    4225             :       assert(OVT.isInteger()
    4226             :         && "VAARG promotion is supported only for vectors or integer types");
    4227           1 :       TruncOp = ISD::TRUNCATE;
    4228             :     }
    4229             : 
    4230             :     // Perform the larger operation, then convert back
    4231           4 :     Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
    4232           3 :              Node->getConstantOperandVal(3));
    4233           2 :     Chain = Tmp1.getValue(1);
    4234             : 
    4235           3 :     Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
    4236             : 
    4237             :     // Modified the chain result - switch anything that used the old chain to
    4238             :     // use the new one.
    4239           2 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
    4240           2 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
    4241           1 :     if (UpdatedNodes) {
    4242           0 :       UpdatedNodes->insert(Tmp2.getNode());
    4243           0 :       UpdatedNodes->insert(Chain.getNode());
    4244             :     }
    4245           1 :     ReplacedNode(Node);
    4246             :     break;
    4247             :   }
    4248        1976 :   case ISD::MUL:
    4249             :   case ISD::SDIV:
    4250             :   case ISD::SREM:
    4251             :   case ISD::UDIV:
    4252             :   case ISD::UREM:
    4253             :   case ISD::AND:
    4254             :   case ISD::OR:
    4255             :   case ISD::XOR: {
    4256             :     unsigned ExtOp, TruncOp;
    4257        1976 :     if (OVT.isVector()) {
    4258             :       ExtOp   = ISD::BITCAST;
    4259             :       TruncOp = ISD::BITCAST;
    4260             :     } else {
    4261             :       assert(OVT.isInteger() && "Cannot promote logic operation");
    4262             : 
    4263          10 :       switch (Node->getOpcode()) {
    4264             :       default:
    4265             :         ExtOp = ISD::ANY_EXTEND;
    4266             :         break;
    4267           3 :       case ISD::SDIV:
    4268             :       case ISD::SREM:
    4269           3 :         ExtOp = ISD::SIGN_EXTEND;
    4270           3 :         break;
    4271           6 :       case ISD::UDIV:
    4272             :       case ISD::UREM:
    4273           6 :         ExtOp = ISD::ZERO_EXTEND;
    4274           6 :         break;
    4275             :       }
    4276             :       TruncOp = ISD::TRUNCATE;
    4277             :     }
    4278             :     // Promote each of the values to the new type.
    4279        7904 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4280        7904 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4281             :     // Perform the larger operation, then convert back
    4282        7904 :     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
    4283        5928 :     Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
    4284        1976 :     break;
    4285             :   }
    4286           0 :   case ISD::UMUL_LOHI:
    4287             :   case ISD::SMUL_LOHI: {
    4288             :     // Promote to a multiply in a wider integer type.
    4289           0 :     unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
    4290           0 :                                                          : ISD::SIGN_EXTEND;
    4291           0 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4292           0 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4293           0 :     Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
    4294             : 
    4295           0 :     auto &DL = DAG.getDataLayout();
    4296           0 :     unsigned OriginalSize = OVT.getScalarSizeInBits();
    4297           0 :     Tmp2 = DAG.getNode(
    4298             :         ISD::SRL, dl, NVT, Tmp1,
    4299           0 :         DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
    4300           0 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
    4301           0 :     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
    4302           0 :     break;
    4303             :   }
    4304         160 :   case ISD::SELECT: {
    4305             :     unsigned ExtOp, TruncOp;
    4306         800 :     if (Node->getValueType(0).isVector() ||
    4307         320 :         Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
    4308             :       ExtOp   = ISD::BITCAST;
    4309             :       TruncOp = ISD::BITCAST;
    4310           8 :     } else if (Node->getValueType(0).isInteger()) {
    4311             :       ExtOp   = ISD::ANY_EXTEND;
    4312             :       TruncOp = ISD::TRUNCATE;
    4313             :     } else {
    4314           4 :       ExtOp   = ISD::FP_EXTEND;
    4315           4 :       TruncOp = ISD::FP_ROUND;
    4316             :     }
    4317         320 :     Tmp1 = Node->getOperand(0);
    4318             :     // Promote each of the values to the new type.
    4319         640 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4320         640 :     Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
    4321             :     // Perform the larger operation, then round down.
    4322         320 :     Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
    4323         160 :     if (TruncOp != ISD::FP_ROUND)
    4324         468 :       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
    4325             :     else
    4326           8 :       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
    4327          12 :                          DAG.getIntPtrConstant(0, dl));
    4328         160 :     Results.push_back(Tmp1);
    4329         160 :     break;
    4330             :   }
    4331         402 :   case ISD::VECTOR_SHUFFLE: {
    4332         804 :     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
    4333             : 
    4334             :     // Cast the two input vectors.
    4335        1608 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
    4336        1608 :     Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
    4337             : 
    4338             :     // Convert the shuffle mask to the right # elements.
    4339         804 :     Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
    4340        1206 :     Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
    4341         402 :     Results.push_back(Tmp1);
    4342             :     break;
    4343             :   }
    4344         177 :   case ISD::SETCC: {
    4345         177 :     unsigned ExtOp = ISD::FP_EXTEND;
    4346         177 :     if (NVT.isInteger()) {
    4347             :       ISD::CondCode CCCode =
    4348         180 :         cast<CondCodeSDNode>(Node->getOperand(2))->get();
    4349          60 :       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    4350             :     }
    4351         708 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
    4352         708 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
    4353         354 :     Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
    4354         531 :                                   Tmp1, Tmp2, Node->getOperand(2)));
    4355         177 :     break;
    4356             :   }
    4357           1 :   case ISD::BR_CC: {
    4358           1 :     unsigned ExtOp = ISD::FP_EXTEND;
    4359           1 :     if (NVT.isInteger()) {
    4360             :       ISD::CondCode CCCode =
    4361           0 :         cast<CondCodeSDNode>(Node->getOperand(1))->get();
    4362           0 :       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    4363             :     }
    4364           4 :     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
    4365           4 :     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
    4366           2 :     Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
    4367           3 :                                   Node->getOperand(0), Node->getOperand(1),
    4368           4 :                                   Tmp1, Tmp2, Node->getOperand(4)));
    4369           1 :     break;
    4370             :   }
    4371         151 :   case ISD::FADD:
    4372             :   case ISD::FSUB:
    4373             :   case ISD::FMUL:
    4374             :   case ISD::FDIV:
    4375             :   case ISD::FREM:
    4376             :   case ISD::FMINNUM:
    4377             :   case ISD::FMAXNUM:
    4378             :   case ISD::FPOW: {
    4379         604 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4380         604 :     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
    4381         302 :     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
    4382         604 :                        Node->getFlags());
    4383         302 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4384         453 :                                   Tmp3, DAG.getIntPtrConstant(0, dl)));
    4385         151 :     break;
    4386             :   }
    4387          17 :   case ISD::FMA: {
    4388          68 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4389          68 :     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
    4390          68 :     Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
    4391          17 :     Results.push_back(
    4392          51 :         DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4393          17 :                     DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
    4394          85 :                     DAG.getIntPtrConstant(0, dl)));
    4395          17 :     break;
    4396             :   }
    4397          12 :   case ISD::FCOPYSIGN:
    4398             :   case ISD::FPOWI: {
    4399          48 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4400          24 :     Tmp2 = Node->getOperand(1);
    4401          48 :     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
    4402             : 
    4403             :     // fcopysign doesn't change anything but the sign bit, so
    4404             :     //   (fp_round (fcopysign (fpext a), b))
    4405             :     // is as precise as
    4406             :     //   (fp_round (fpext a))
    4407             :     // which is a no-op. Mark it as a TRUNCating FP_ROUND.
    4408          12 :     const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
    4409          24 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4410          36 :                                   Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
    4411          12 :     break;
    4412             :   }
    4413         107 :   case ISD::FFLOOR:
    4414             :   case ISD::FCEIL:
    4415             :   case ISD::FRINT:
    4416             :   case ISD::FNEARBYINT:
    4417             :   case ISD::FROUND:
    4418             :   case ISD::FTRUNC:
    4419             :   case ISD::FNEG:
    4420             :   case ISD::FSQRT:
    4421             :   case ISD::FSIN:
    4422             :   case ISD::FCOS:
    4423             :   case ISD::FLOG:
    4424             :   case ISD::FLOG2:
    4425             :   case ISD::FLOG10:
    4426             :   case ISD::FABS:
    4427             :   case ISD::FEXP:
    4428             :   case ISD::FEXP2: {
    4429         428 :     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
    4430         428 :     Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
    4431         214 :     Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
    4432         321 :                                   Tmp2, DAG.getIntPtrConstant(0, dl)));
    4433         107 :     break;
    4434             :   }
    4435         363 :   case ISD::BUILD_VECTOR: {
    4436         363 :     MVT EltVT = OVT.getVectorElementType();
    4437         363 :     MVT NewEltVT = NVT.getVectorElementType();
    4438             : 
    4439             :     // Handle bitcasts to a different vector type with the same total bit size
    4440             :     //
    4441             :     // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
    4442             :     //  =>
    4443             :     //  v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
    4444             : 
    4445             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4446             :            "Invalid promote type for build_vector");
    4447             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4448             : 
    4449         363 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4450             : 
    4451         726 :     SmallVector<SDValue, 8> NewOps;
    4452        1452 :     for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
    4453        1452 :       SDValue Op = Node->getOperand(I);
    4454        3630 :       NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
    4455             :     }
    4456             : 
    4457         726 :     SDLoc SL(Node);
    4458        1452 :     SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
    4459        1089 :     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
    4460         363 :     Results.push_back(CvtVec);
    4461             :     break;
    4462             :   }
    4463        2410 :   case ISD::EXTRACT_VECTOR_ELT: {
    4464        2410 :     MVT EltVT = OVT.getVectorElementType();
    4465        2410 :     MVT NewEltVT = NVT.getVectorElementType();
    4466             : 
    4467             :     // Handle bitcasts to a different vector type with the same total bit size.
    4468             :     //
    4469             :     // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
    4470             :     //  =>
    4471             :     //  v4i32:castx = bitcast x:v2i64
    4472             :     //
    4473             :     // i64 = bitcast
    4474             :     //   (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
    4475             :     //                       (i32 (extract_vector_elt castx, (2 * y + 1)))
    4476             :     //
    4477             : 
    4478             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4479             :            "Invalid promote type for extract_vector_elt");
    4480             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4481             : 
    4482        2410 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4483        2410 :     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
    4484             : 
    4485        4820 :     SDValue Idx = Node->getOperand(1);
    4486        4820 :     EVT IdxVT = Idx.getValueType();
    4487        4820 :     SDLoc SL(Node);
    4488        2410 :     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
    4489        4820 :     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
    4490             : 
    4491        9640 :     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
    4492             : 
    4493        4820 :     SmallVector<SDValue, 8> NewOps;
    4494        7230 :     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
    4495        4820 :       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
    4496        9640 :       SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
    4497             : 
    4498        4820 :       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
    4499        9640 :                                 CastVec, TmpIdx);
    4500        4820 :       NewOps.push_back(Elt);
    4501             :     }
    4502             : 
    4503        7230 :     SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
    4504        7230 :     Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
    4505             :     break;
    4506             :   }
    4507           4 :   case ISD::INSERT_VECTOR_ELT: {
    4508           4 :     MVT EltVT = OVT.getVectorElementType();
    4509           4 :     MVT NewEltVT = NVT.getVectorElementType();
    4510             : 
    4511             :     // Handle bitcasts to a different vector type with the same total bit size
    4512             :     //
    4513             :     // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
    4514             :     //  =>
    4515             :     //  v4i32:castx = bitcast x:v2i64
    4516             :     //  v2i32:casty = bitcast y:i64
    4517             :     //
    4518             :     // v2i64 = bitcast
    4519             :     //   (v4i32 insert_vector_elt
    4520             :     //       (v4i32 insert_vector_elt v4i32:castx,
    4521             :     //                                (extract_vector_elt casty, 0), 2 * z),
    4522             :     //        (extract_vector_elt casty, 1), (2 * z + 1))
    4523             : 
    4524             :     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
    4525             :            "Invalid promote type for insert_vector_elt");
    4526             :     assert(NewEltVT.bitsLT(EltVT) && "not handled");
    4527             : 
    4528           4 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4529           4 :     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
    4530             : 
    4531           8 :     SDValue Val = Node->getOperand(1);
    4532           8 :     SDValue Idx = Node->getOperand(2);
    4533           8 :     EVT IdxVT = Idx.getValueType();
    4534           8 :     SDLoc SL(Node);
    4535             : 
    4536          12 :     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
    4537           8 :     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
    4538             : 
    4539          16 :     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
    4540          12 :     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
    4541             : 
    4542           4 :     SDValue NewVec = CastVec;
    4543          12 :     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
    4544           8 :       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
    4545          16 :       SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
    4546             : 
    4547           8 :       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
    4548          16 :                                 CastVal, IdxOffset);
    4549             : 
    4550          16 :       NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
    4551          16 :                            NewVec, Elt, InEltIdx);
    4552             :     }
    4553             : 
    4554          12 :     Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
    4555             :     break;
    4556             :   }
    4557          28 :   case ISD::SCALAR_TO_VECTOR: {
    4558          28 :     MVT EltVT = OVT.getVectorElementType();
    4559          28 :     MVT NewEltVT = NVT.getVectorElementType();
    4560             : 
    4561             :     // Handle bitcasts to different vector type with the same total bit size.
    4562             :     //
    4563             :     // e.g. v2i64 = scalar_to_vector x:i64
    4564             :     //   =>
    4565             :     //  concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
    4566             :     //
    4567             : 
    4568          28 :     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
    4569          56 :     SDValue Val = Node->getOperand(0);
    4570          56 :     SDLoc SL(Node);
    4571             : 
    4572          84 :     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
    4573          56 :     SDValue Undef = DAG.getUNDEF(MidVT);
    4574             : 
    4575          56 :     SmallVector<SDValue, 8> NewElts;
    4576          28 :     NewElts.push_back(CastVal);
    4577          56 :     for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
    4578          28 :       NewElts.push_back(Undef);
    4579             : 
    4580         112 :     SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
    4581          84 :     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
    4582          28 :     Results.push_back(CvtVec);
    4583             :     break;
    4584             :   }
    4585             :   }
    4586             : 
    4587             :   // Replace the original node with the legalized result.
    4588        6124 :   if (!Results.empty())
    4589       12038 :     ReplaceNode(Node, Results.data());
    4590        6124 : }
    4591             : 
    4592             : /// This is the entry point for the file.
    4593      280070 : void SelectionDAG::Legalize() {
    4594      280070 :   AssignTopologicalOrder();
    4595             : 
    4596      560139 :   SmallPtrSet<SDNode *, 16> LegalizedNodes;
    4597             :   // Use a delete listener to remove nodes which were deleted during
    4598             :   // legalization from LegalizeNodes. This is needed to handle the situation
    4599             :   // where a new node is allocated by the object pool to the same address of a
    4600             :   // previously deleted node.
    4601             :   DAGNodeDeletedListener DeleteListener(
    4602             :       *this,
    4603     1400867 :       [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
    4604             : 
    4605             :   SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
    4606             : 
    4607             :   // Visit all the nodes. We start in topological order, so that we see
    4608             :   // nodes with their original operands intact. Legalization can produce
    4609             :   // new nodes which may themselves need to be legalized. Iterate until all
    4610             :   // nodes have been legalized.
    4611             :   for (;;) {
    4612      798510 :     bool AnyLegalized = false;
    4613    29405352 :     for (auto NI = allnodes_end(); NI != allnodes_begin();) {
    4614    27808333 :       --NI;
    4615             : 
    4616    27808333 :       SDNode *N = &*NI;
    4617    28242196 :       if (N->use_empty() && N != getRoot().getNode()) {
    4618      433863 :         ++NI;
    4619      433863 :         DeleteNode(N);
    4620      433863 :         continue;
    4621             :       }
    4622             : 
    4623    27374470 :       if (LegalizedNodes.insert(N).second) {
    4624    10460895 :         AnyLegalized = true;
    4625    10460895 :         Legalizer.LegalizeOp(N);
    4626             : 
    4627    10460894 :         if (N->use_empty() && N != getRoot().getNode()) {
    4628     1048515 :           ++NI;
    4629     1048515 :           DeleteNode(N);
    4630             :         }
    4631             :       }
    4632             :     }
    4633      798509 :     if (!AnyLegalized)
    4634             :       break;
    4635             : 
    4636             :   }
    4637             : 
    4638             :   // Remove dead nodes now.
    4639      280069 :   RemoveDeadNodes();
    4640      280069 : }
    4641             : 
    4642    10371258 : bool SelectionDAG::LegalizeOp(SDNode *N,
    4643             :                               SmallSetVector<SDNode *, 16> &UpdatedNodes) {
    4644    20742516 :   SmallPtrSet<SDNode *, 16> LegalizedNodes;
    4645    10371258 :   SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
    4646             : 
    4647             :   // Directly insert the node in question, and legalize it. This will recurse
    4648             :   // as needed through operands.
    4649    10371258 :   LegalizedNodes.insert(N);
    4650    10371258 :   Legalizer.LegalizeOp(N);
    4651             : 
    4652    20742516 :   return LegalizedNodes.count(N);
    4653             : }

Generated by: LCOV version 1.13