LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeIntegerTypes.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1679 1840 91.2 %
Date: 2018-06-17 00:07:59 Functions: 121 130 93.1 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements integer type expansion and promotion for LegalizeTypes.
      11             : // Promotion is the act of changing a computation in an illegal type into a
      12             : // computation in a larger type.  For example, implementing i8 arithmetic in an
      13             : // i32 register (often needed on powerpc).
      14             : // Expansion is the act of changing a computation in an illegal type into a
      15             : // computation in two identical registers of a smaller type.  For example,
      16             : // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
      17             : // targets).
      18             : //
      19             : //===----------------------------------------------------------------------===//
      20             : 
      21             : #include "LegalizeTypes.h"
      22             : #include "llvm/IR/DerivedTypes.h"
      23             : #include "llvm/Support/ErrorHandling.h"
      24             : #include "llvm/Support/KnownBits.h"
      25             : #include "llvm/Support/raw_ostream.h"
      26             : using namespace llvm;
      27             : 
      28             : #define DEBUG_TYPE "legalize-types"
      29             : 
      30             : //===----------------------------------------------------------------------===//
      31             : //  Integer Result Promotion
      32             : //===----------------------------------------------------------------------===//
      33             : 
      34             : /// PromoteIntegerResult - This method is called when a result of a node is
      35             : /// found to be in need of promotion to a larger type.  At this point, the node
      36             : /// may also have invalid operands or may have other results that need
      37             : /// expansion, we just know that (at least) one result needs promotion.
      38      148424 : void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
      39             :   LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
      40             :              dbgs() << "\n");
      41      148424 :   SDValue Res = SDValue();
      42             : 
      43             :   // See if the target wants to custom expand this node.
      44      296848 :   if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
      45             :     LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
      46         369 :     return;
      47             :   }
      48             : 
      49      296110 :   switch (N->getOpcode()) {
      50           0 :   default:
      51             : #ifndef NDEBUG
      52             :     dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
      53             :     N->dump(&DAG); dbgs() << "\n";
      54             : #endif
      55           0 :     llvm_unreachable("Do not know how to promote this operator!");
      56           0 :   case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
      57          63 :   case ISD::AssertSext:  Res = PromoteIntRes_AssertSext(N); break;
      58          63 :   case ISD::AssertZext:  Res = PromoteIntRes_AssertZext(N); break;
      59        5271 :   case ISD::BITCAST:     Res = PromoteIntRes_BITCAST(N); break;
      60          16 :   case ISD::BITREVERSE:  Res = PromoteIntRes_BITREVERSE(N); break;
      61          21 :   case ISD::BSWAP:       Res = PromoteIntRes_BSWAP(N); break;
      62           0 :   case ISD::BUILD_PAIR:  Res = PromoteIntRes_BUILD_PAIR(N); break;
      63       13416 :   case ISD::Constant:    Res = PromoteIntRes_Constant(N); break;
      64           8 :   case ISD::CTLZ_ZERO_UNDEF:
      65           8 :   case ISD::CTLZ:        Res = PromoteIntRes_CTLZ(N); break;
      66          92 :   case ISD::CTPOP:       Res = PromoteIntRes_CTPOP(N); break;
      67          35 :   case ISD::CTTZ_ZERO_UNDEF:
      68          35 :   case ISD::CTTZ:        Res = PromoteIntRes_CTTZ(N); break;
      69        7246 :   case ISD::EXTRACT_VECTOR_ELT:
      70        7246 :                          Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
      71        5954 :   case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
      72           4 :   case ISD::MLOAD:       Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
      73           4 :     break;
      74           0 :   case ISD::MGATHER:     Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
      75           0 :     break;
      76         750 :   case ISD::SELECT:      Res = PromoteIntRes_SELECT(N); break;
      77           7 :   case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
      78           0 :   case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
      79       51036 :   case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
      80          96 :   case ISD::SMIN:
      81          96 :   case ISD::SMAX:        Res = PromoteIntRes_SExtIntBinOp(N); break;
      82          50 :   case ISD::UMIN:
      83          50 :   case ISD::UMAX:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
      84             : 
      85         926 :   case ISD::SHL:         Res = PromoteIntRes_SHL(N); break;
      86          58 :   case ISD::SIGN_EXTEND_INREG:
      87          58 :                          Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
      88          91 :   case ISD::SRA:         Res = PromoteIntRes_SRA(N); break;
      89        5182 :   case ISD::SRL:         Res = PromoteIntRes_SRL(N); break;
      90       35373 :   case ISD::TRUNCATE:    Res = PromoteIntRes_TRUNCATE(N); break;
      91        2355 :   case ISD::UNDEF:       Res = PromoteIntRes_UNDEF(N); break;
      92          37 :   case ISD::VAARG:       Res = PromoteIntRes_VAARG(N); break;
      93             : 
      94        2472 :   case ISD::EXTRACT_SUBVECTOR:
      95        2472 :                          Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
      96         108 :   case ISD::VECTOR_SHUFFLE:
      97         108 :                          Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
      98         207 :   case ISD::INSERT_VECTOR_ELT:
      99         207 :                          Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
     100        1203 :   case ISD::BUILD_VECTOR:
     101        1203 :                          Res = PromoteIntRes_BUILD_VECTOR(N); break;
     102          27 :   case ISD::SCALAR_TO_VECTOR:
     103          27 :                          Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
     104         177 :   case ISD::CONCAT_VECTORS:
     105         177 :                          Res = PromoteIntRes_CONCAT_VECTORS(N); break;
     106             : 
     107           3 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     108             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     109             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     110           3 :                          Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
     111             : 
     112        2609 :   case ISD::SIGN_EXTEND:
     113             :   case ISD::ZERO_EXTEND:
     114        2609 :   case ISD::ANY_EXTEND:  Res = PromoteIntRes_INT_EXTEND(N); break;
     115             : 
     116         177 :   case ISD::FP_TO_SINT:
     117         177 :   case ISD::FP_TO_UINT:  Res = PromoteIntRes_FP_TO_XINT(N); break;
     118             : 
     119         888 :   case ISD::FP_TO_FP16:  Res = PromoteIntRes_FP_TO_FP16(N); break;
     120             : 
     121        8303 :   case ISD::AND:
     122             :   case ISD::OR:
     123             :   case ISD::XOR:
     124             :   case ISD::ADD:
     125             :   case ISD::SUB:
     126        8303 :   case ISD::MUL:         Res = PromoteIntRes_SimpleIntBinOp(N); break;
     127             : 
     128         298 :   case ISD::SDIV:
     129         298 :   case ISD::SREM:        Res = PromoteIntRes_SExtIntBinOp(N); break;
     130             : 
     131         252 :   case ISD::UDIV:
     132         252 :   case ISD::UREM:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
     133             : 
     134         569 :   case ISD::SADDO:
     135         569 :   case ISD::SSUBO:       Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
     136         483 :   case ISD::UADDO:
     137         483 :   case ISD::USUBO:       Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
     138         102 :   case ISD::SMULO:
     139         102 :   case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;
     140             : 
     141          15 :   case ISD::ADDCARRY:
     142          15 :   case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
     143             : 
     144             :   case ISD::ATOMIC_LOAD:
     145         126 :     Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
     146             : 
     147             :   case ISD::ATOMIC_LOAD_ADD:
     148             :   case ISD::ATOMIC_LOAD_SUB:
     149             :   case ISD::ATOMIC_LOAD_AND:
     150             :   case ISD::ATOMIC_LOAD_CLR:
     151             :   case ISD::ATOMIC_LOAD_OR:
     152             :   case ISD::ATOMIC_LOAD_XOR:
     153             :   case ISD::ATOMIC_LOAD_NAND:
     154             :   case ISD::ATOMIC_LOAD_MIN:
     155             :   case ISD::ATOMIC_LOAD_MAX:
     156             :   case ISD::ATOMIC_LOAD_UMIN:
     157             :   case ISD::ATOMIC_LOAD_UMAX:
     158             :   case ISD::ATOMIC_SWAP:
     159        1113 :     Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
     160             : 
     161             :   case ISD::ATOMIC_CMP_SWAP:
     162             :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     163         773 :     Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
     164         773 :     break;
     165             :   }
     166             : 
     167             :   // If the result is null then the sub-method took care of registering it.
     168      148055 :   if (Res.getNode())
     169      148055 :     SetPromotedInteger(SDValue(N, ResNo), Res);
     170             : }
     171             : 
     172           0 : SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
     173             :                                                      unsigned ResNo) {
     174           0 :   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
     175           0 :   return GetPromotedInteger(Op);
     176             : }
     177             : 
     178          63 : SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
     179             :   // Sign-extend the new bits, and continue the assertion.
     180         126 :   SDValue Op = SExtPromotedInteger(N->getOperand(0));
     181         126 :   return DAG.getNode(ISD::AssertSext, SDLoc(N),
     182         252 :                      Op.getValueType(), Op, N->getOperand(1));
     183             : }
     184             : 
     185          63 : SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
     186             :   // Zero the new bits, and continue the assertion.
     187         126 :   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
     188         126 :   return DAG.getNode(ISD::AssertZext, SDLoc(N),
     189         252 :                      Op.getValueType(), Op, N->getOperand(1));
     190             : }
     191             : 
     192         126 : SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
     193         378 :   EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     194         252 :   SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
     195             :                               N->getMemoryVT(), ResVT,
     196             :                               N->getChain(), N->getBasePtr(),
     197         378 :                               N->getMemOperand());
     198             :   // Legalize the chain result - switch anything that used the old chain to
     199             :   // use the new one.
     200         126 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     201         126 :   return Res;
     202             : }
     203             : 
     204        1113 : SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
     205        2226 :   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
     206        2226 :   SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
     207             :                               N->getMemoryVT(),
     208             :                               N->getChain(), N->getBasePtr(),
     209        3339 :                               Op2, N->getMemOperand());
     210             :   // Legalize the chain result - switch anything that used the old chain to
     211             :   // use the new one.
     212        1113 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     213        1113 :   return Res;
     214             : }
     215             : 
     216         773 : SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
     217             :                                                       unsigned ResNo) {
     218         773 :   if (ResNo == 1) {
     219             :     assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
     220        1220 :     EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
     221        1220 :     EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
     222             : 
     223             :     // Only use the result of getSetCCResultType if it is legal,
     224             :     // otherwise just use the promoted result type (NVT).
     225         610 :     if (!TLI.isTypeLegal(SVT))
     226           0 :       SVT = NVT;
     227             : 
     228        1830 :     SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
     229         610 :     SDValue Res = DAG.getAtomicCmpSwap(
     230         610 :         ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
     231         610 :         N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
     232        1220 :         N->getMemOperand());
     233         610 :     ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
     234         610 :     ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
     235             :     return Res.getValue(1);
     236             :   }
     237             : 
     238         326 :   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
     239         326 :   SDValue Op3 = GetPromotedInteger(N->getOperand(3));
     240             :   SDVTList VTs =
     241         652 :       DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
     242         163 :   SDValue Res = DAG.getAtomicCmpSwap(
     243         163 :       N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
     244         489 :       N->getBasePtr(), Op2, Op3, N->getMemOperand());
     245             :   // Update the use to N with the newly created Res.
     246         978 :   for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
     247         326 :     ReplaceValueWith(SDValue(N, i), Res.getValue(i));
     248         163 :   return Res;
     249             : }
     250             : 
     251        5271 : SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
     252        5271 :   SDValue InOp = N->getOperand(0);
     253             :   EVT InVT = InOp.getValueType();
     254       10542 :   EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
     255       10542 :   EVT OutVT = N->getValueType(0);
     256       10542 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
     257             :   SDLoc dl(N);
     258             : 
     259       10542 :   switch (getTypeAction(InVT)) {
     260             :   case TargetLowering::TypeLegal:
     261             :     break;
     262         485 :   case TargetLowering::TypePromoteInteger:
     263         554 :     if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
     264             :       // The input promotes to the same size.  Convert the promoted value.
     265           0 :       return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
     266             :     break;
     267           0 :   case TargetLowering::TypeSoftenFloat:
     268             :     // Promote the integer operand by hand.
     269           0 :     return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
     270         712 :   case TargetLowering::TypePromoteFloat: {
     271             :     // Convert the promoted float by hand.
     272         712 :     SDValue PromotedOp = GetPromotedFloat(InOp);
     273        1424 :     return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
     274             :     break;
     275             :   }
     276             :   case TargetLowering::TypeExpandInteger:
     277             :   case TargetLowering::TypeExpandFloat:
     278             :     break;
     279             :   case TargetLowering::TypeScalarizeVector:
     280             :     // Convert the element to an integer and promote it by hand.
     281        2540 :     if (!NOutVT.isVector())
     282        2503 :       return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
     283        2503 :                          BitConvertToInteger(GetScalarizedVector(InOp)));
     284             :     break;
     285         488 :   case TargetLowering::TypeSplitVector: {
     286             :     // For example, i32 = BITCAST v2i16 on alpha.  Convert the split
     287             :     // pieces of the input into integers and reassemble in the final type.
     288         488 :     SDValue Lo, Hi;
     289         976 :     GetSplitVector(N->getOperand(0), Lo, Hi);
     290         488 :     Lo = BitConvertToInteger(Lo);
     291         488 :     Hi = BitConvertToInteger(Hi);
     292             : 
     293         976 :     if (DAG.getDataLayout().isBigEndian())
     294             :       std::swap(Lo, Hi);
     295             : 
     296         976 :     InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
     297         488 :                        EVT::getIntegerVT(*DAG.getContext(),
     298             :                                          NOutVT.getSizeInBits()),
     299        1464 :                        JoinIntegers(Lo, Hi));
     300         976 :     return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
     301             :   }
     302          11 :   case TargetLowering::TypeWidenVector:
     303             :     // The input is widened to the same size. Convert to the widened value.
     304             :     // Make sure that the outgoing value is not a vector, because this would
     305             :     // make us bitcast between two vectors which are legalized in different ways.
     306          22 :     if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
     307          18 :       return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
     308             :   }
     309             : 
     310        1559 :   return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
     311        1559 :                      CreateStackStoreLoad(InOp, OutVT));
     312             : }
     313             : 
     314          21 : SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
     315          42 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     316          42 :   EVT OVT = N->getValueType(0);
     317          42 :   EVT NVT = Op.getValueType();
     318             :   SDLoc dl(N);
     319             : 
     320          21 :   unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
     321          21 :   return DAG.getNode(
     322          21 :       ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
     323             :       DAG.getConstant(DiffBits, dl,
     324          84 :                       TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
     325             : }
     326             : 
     327          16 : SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
     328          32 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     329          32 :   EVT OVT = N->getValueType(0);
     330          32 :   EVT NVT = Op.getValueType();
     331             :   SDLoc dl(N);
     332             : 
     333          16 :   unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
     334          16 :   return DAG.getNode(
     335          16 :       ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
     336             :       DAG.getConstant(DiffBits, dl,
     337          64 :                       TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
     338             : }
     339             : 
     340           0 : SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
     341             :   // The pair element type may be legal, or may not promote to the same type as
     342             :   // the result, for example i14 = BUILD_PAIR (i7, i7).  Handle all cases.
     343           0 :   return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
     344           0 :                      TLI.getTypeToTransformTo(*DAG.getContext(),
     345             :                      N->getValueType(0)), JoinIntegers(N->getOperand(0),
     346           0 :                      N->getOperand(1)));
     347             : }
     348             : 
     349       13416 : SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
     350       26832 :   EVT VT = N->getValueType(0);
     351             :   // FIXME there is no actual debug info here
     352             :   SDLoc dl(N);
     353             :   // Zero extend things like i1, sign extend everything else.  It shouldn't
     354             :   // matter in theory which one we pick, but this tends to give better code?
     355       13416 :   unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
     356       13416 :   SDValue Result = DAG.getNode(Opc, dl,
     357       26832 :                                TLI.getTypeToTransformTo(*DAG.getContext(), VT),
     358       26832 :                                SDValue(N, 0));
     359             :   assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
     360       26832 :   return Result;
     361             : }
     362             : 
     363           8 : SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
     364             :   // Zero extend to the promoted type and do the count there.
     365          16 :   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
     366             :   SDLoc dl(N);
     367          16 :   EVT OVT = N->getValueType(0);
     368           8 :   EVT NVT = Op.getValueType();
     369          24 :   Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
     370             :   // Subtract off the extra leading bits in the bigger type.
     371           8 :   return DAG.getNode(
     372             :       ISD::SUB, dl, NVT, Op,
     373           8 :       DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
     374          16 :                       NVT));
     375             : }
     376             : 
     377          92 : SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
     378             :   // Zero extend to the promoted type and do the count there.
     379         184 :   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
     380         368 :   return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
     381             : }
     382             : 
     383          35 : SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
     384          70 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     385          70 :   EVT OVT = N->getValueType(0);
     386          35 :   EVT NVT = Op.getValueType();
     387             :   SDLoc dl(N);
     388          35 :   if (N->getOpcode() == ISD::CTTZ) {
     389             :     // The count is the same in the promoted type except if the original
     390             :     // value was zero.  This can be handled by setting the bit just off
     391             :     // the top of the original type.
     392             :     auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
     393          15 :                                       OVT.getScalarSizeInBits());
     394          30 :     Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
     395             :   }
     396         140 :   return DAG.getNode(N->getOpcode(), dl, NVT, Op);
     397             : }
     398             : 
     399        7246 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
     400             :   SDLoc dl(N);
     401       21738 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     402        7246 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
     403       21738 :                      N->getOperand(1));
     404             : }
     405             : 
     406         177 : SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
     407         531 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     408         177 :   unsigned NewOpc = N->getOpcode();
     409             :   SDLoc dl(N);
     410             : 
     411             :   // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
     412             :   // not Legal, check to see if we can use FP_TO_SINT instead.  (If both UINT
     413             :   // and SINT conversions are Custom, there is no way to tell which is
     414             :   // preferable. We choose SINT because that's the right thing on PPC.)
     415         177 :   if (N->getOpcode() == ISD::FP_TO_UINT &&
     416         177 :       !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
     417             :       TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
     418             :     NewOpc = ISD::FP_TO_SINT;
     419             : 
     420         531 :   SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
     421             : 
     422             :   // Assert that the converted value fits in the original type.  If it doesn't
     423             :   // (eg: because the value being converted is too big), then the result of the
     424             :   // original operation was undefined anyway, so the assert is still correct.
     425             :   //
     426             :   // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
     427             :   //   before legalization: fp-to-uint16, 65534. -> 0xfffe
     428             :   //   after legalization: fp-to-sint32, 65534. -> 0x0000fffe
     429         354 :   return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
     430             :                      ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
     431         708 :                      DAG.getValueType(N->getValueType(0).getScalarType()));
     432             : }
     433             : 
     434         888 : SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
     435        2664 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     436             :   SDLoc dl(N);
     437             : 
     438        4440 :   return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
     439             : }
     440             : 
     441        2609 : SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
     442        7827 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     443             :   SDLoc dl(N);
     444             : 
     445        7827 :   if (getTypeAction(N->getOperand(0).getValueType())
     446             :       == TargetLowering::TypePromoteInteger) {
     447        3212 :     SDValue Res = GetPromotedInteger(N->getOperand(0));
     448             :     assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
     449             : 
     450             :     // If the result and operand types are the same after promotion, simplify
     451             :     // to an in-register extension.
     452           4 :     if (NVT == Res.getValueType()) {
     453             :       // The high bits are not guaranteed to be anything.  Insert an extend.
     454        3168 :       if (N->getOpcode() == ISD::SIGN_EXTEND)
     455         404 :         return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
     456         808 :                            DAG.getValueType(N->getOperand(0).getValueType()));
     457        1180 :       if (N->getOpcode() == ISD::ZERO_EXTEND)
     458         798 :         return DAG.getZeroExtendInReg(Res, dl,
     459        1596 :                       N->getOperand(0).getValueType().getScalarType());
     460             :       assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
     461         382 :       return Res;
     462             :     }
     463             :   }
     464             : 
     465             :   // Otherwise, just extend the original operand all the way to the larger type.
     466        4100 :   return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
     467             : }
     468             : 
     469        5954 : SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
     470             :   assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
     471       17862 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     472             :   ISD::LoadExtType ExtType =
     473             :     ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
     474             :   SDLoc dl(N);
     475        5954 :   SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
     476       11908 :                                N->getMemoryVT(), N->getMemOperand());
     477             : 
     478             :   // Legalize the chain result - switch anything that used the old chain to
     479             :   // use the new one.
     480        5954 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     481       11908 :   return Res;
     482             : }
     483             : 
     484           4 : SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
     485          12 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     486           4 :   SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
     487             : 
     488             :   SDLoc dl(N);
     489           4 :   SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
     490             :                                   N->getMask(), ExtSrc0, N->getMemoryVT(),
     491           8 :                                   N->getMemOperand(), ISD::SEXTLOAD);
     492             :   // Legalize the chain result - switch anything that used the old chain to
     493             :   // use the new one.
     494           4 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     495           8 :   return Res;
     496             : }
     497             : 
     498           0 : SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
     499           0 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     500           0 :   SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
     501             :   assert(NVT == ExtSrc0.getValueType() &&
     502             :       "Gather result type and the passThru agrument type should be the same");
     503             : 
     504             :   SDLoc dl(N);
     505             :   SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
     506           0 :                    N->getIndex(), N->getScale() };
     507           0 :   SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
     508             :                                     N->getMemoryVT(), dl, Ops,
     509           0 :                                     N->getMemOperand());
     510             :   // Legalize the chain result - switch anything that used the old chain to
     511             :   // use the new one.
     512           0 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     513           0 :   return Res;
     514             : }
     515             : 
     516             : /// Promote the overflow flag of an overflowing arithmetic node.
     517        1156 : SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
     518             :   // Simply change the return type of the boolean result.
     519        2312 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
     520        2312 :   EVT ValueVTs[] = { N->getValueType(0), NVT };
     521        1156 :   SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
     522        1156 :   unsigned NumOps = N->getNumOperands();
     523             :   assert(NumOps <= 3 && "Too many operands");
     524        1156 :   if (NumOps == 3)
     525          10 :     Ops[2] = N->getOperand(2);
     526             : 
     527        2312 :   SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
     528        5780 :                             DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
     529             : 
     530             :   // Modified the sum result - switch anything that used the old sum to use
     531             :   // the new one.
     532        1156 :   ReplaceValueWith(SDValue(N, 0), Res);
     533             : 
     534        1156 :   return SDValue(Res.getNode(), 1);
     535             : }
     536             : 
     537         569 : SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
     538         569 :   if (ResNo == 1)
     539         567 :     return PromoteIntRes_Overflow(N);
     540             : 
     541             :   // The operation overflowed iff the result in the larger type is not the
     542             :   // sign extension of its truncation to the original type.
     543           4 :   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
     544           4 :   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
     545           4 :   EVT OVT = N->getOperand(0).getValueType();
     546           4 :   EVT NVT = LHS.getValueType();
     547             :   SDLoc dl(N);
     548             : 
     549             :   // Do the arithmetic in the larger type.
     550           2 :   unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
     551           4 :   SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
     552             : 
     553             :   // Calculate the overflow flag: sign extend the arithmetic result from
     554             :   // the original type.
     555           2 :   SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
     556           2 :                             DAG.getValueType(OVT));
     557             :   // Overflowed if and only if this is not equal to Res.
     558           4 :   Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
     559             : 
     560             :   // Use the calculated overflow everywhere.
     561           2 :   ReplaceValueWith(SDValue(N, 1), Ofl);
     562             : 
     563           2 :   return Res;
     564             : }
     565             : 
     566         750 : SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
     567        1500 :   SDValue LHS = GetPromotedInteger(N->getOperand(1));
     568        1500 :   SDValue RHS = GetPromotedInteger(N->getOperand(2));
     569        1500 :   return DAG.getSelect(SDLoc(N),
     570        3000 :                        LHS.getValueType(), N->getOperand(0), LHS, RHS);
     571             : }
     572             : 
     573           7 : SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
     574           7 :   SDValue Mask = N->getOperand(0);
     575             : 
     576           7 :   SDValue LHS = GetPromotedInteger(N->getOperand(1));
     577          14 :   SDValue RHS = GetPromotedInteger(N->getOperand(2));
     578          14 :   return DAG.getNode(ISD::VSELECT, SDLoc(N),
     579          21 :                      LHS.getValueType(), Mask, LHS, RHS);
     580             : }
     581             : 
     582           0 : SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
     583           0 :   SDValue LHS = GetPromotedInteger(N->getOperand(2));
     584           0 :   SDValue RHS = GetPromotedInteger(N->getOperand(3));
     585           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
     586             :                      LHS.getValueType(), N->getOperand(0),
     587           0 :                      N->getOperand(1), LHS, RHS, N->getOperand(4));
     588             : }
     589             : 
     590       51036 : SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
     591      102072 :   EVT InVT = N->getOperand(0).getValueType();
     592      153108 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     593             : 
     594       51036 :   EVT SVT = getSetCCResultType(InVT);
     595             : 
     596             :   // If we got back a type that needs to be promoted, this likely means the
     597             :   // the input type also needs to be promoted. So get the promoted type for
     598             :   // the input and try the query again.
     599      102072 :   if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
     600         248 :     if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
     601         230 :       InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
     602         115 :       SVT = getSetCCResultType(InVT);
     603             :     } else {
     604             :       // Input type isn't promoted, just use the default promoted type.
     605           9 :       SVT = NVT;
     606             :     }
     607             :   }
     608             : 
     609             :   SDLoc dl(N);
     610             :   assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
     611             :          "Vector compare must return a vector result!");
     612             : 
     613             :   // Get the SETCC result using the canonical SETCC type.
     614       51036 :   SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
     615      153108 :                               N->getOperand(1), N->getOperand(2));
     616             : 
     617             :   // Convert to the expected type.
     618      102072 :   return DAG.getSExtOrTrunc(SetCC, dl, NVT);
     619             : }
     620             : 
     621         926 : SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
     622        1852 :   SDValue LHS = GetPromotedInteger(N->getOperand(0));
     623         926 :   SDValue RHS = N->getOperand(1);
     624        1852 :   if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
     625         181 :     RHS = ZExtPromotedInteger(RHS);
     626        3704 :   return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
     627             : }
     628             : 
     629          58 : SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
     630         116 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     631         116 :   return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
     632         232 :                      Op.getValueType(), Op, N->getOperand(1));
     633             : }
     634             : 
     635        8303 : SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
     636             :   // The input may have strange things in the top bits of the registers, but
     637             :   // these operations don't care.  They may have weird bits going out, but
     638             :   // that too is okay if they are integer operations.
     639       16606 :   SDValue LHS = GetPromotedInteger(N->getOperand(0));
     640       16606 :   SDValue RHS = GetPromotedInteger(N->getOperand(1));
     641       16606 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     642       33212 :                      LHS.getValueType(), LHS, RHS);
     643             : }
     644             : 
     645         394 : SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
     646             :   // Sign extend the input.
     647         788 :   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
     648         788 :   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
     649         788 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     650        1576 :                      LHS.getValueType(), LHS, RHS);
     651             : }
     652             : 
     653         302 : SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
     654             :   // Zero extend the input.
     655         604 :   SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
     656         604 :   SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
     657         604 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     658        1208 :                      LHS.getValueType(), LHS, RHS);
     659             : }
     660             : 
     661          91 : SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
     662             :   // The input value must be properly sign extended.
     663         182 :   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
     664          91 :   SDValue RHS = N->getOperand(1);
     665         182 :   if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
     666          42 :     RHS = ZExtPromotedInteger(RHS);
     667         364 :   return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
     668             : }
     669             : 
     670        5182 : SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
     671             :   // The input value must be properly zero extended.
     672       10364 :   SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
     673        5182 :   SDValue RHS = N->getOperand(1);
     674       10364 :   if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
     675        1560 :     RHS = ZExtPromotedInteger(RHS);
     676       20728 :   return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
     677             : }
     678             : 
     679       35373 : SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
     680      106119 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     681       35373 :   SDValue Res;
     682       35373 :   SDValue InOp = N->getOperand(0);
     683             :   SDLoc dl(N);
     684             : 
     685       70746 :   switch (getTypeAction(InOp.getValueType())) {
     686           0 :   default: llvm_unreachable("Unknown type action!");
     687       30666 :   case TargetLowering::TypeLegal:
     688             :   case TargetLowering::TypeExpandInteger:
     689       30666 :     Res = InOp;
     690       30666 :     break;
     691        4686 :   case TargetLowering::TypePromoteInteger:
     692        4686 :     Res = GetPromotedInteger(InOp);
     693        4686 :     break;
     694          20 :   case TargetLowering::TypeSplitVector: {
     695          20 :     EVT InVT = InOp.getValueType();
     696             :     assert(InVT.isVector() && "Cannot split scalar types");
     697          20 :     unsigned NumElts = InVT.getVectorNumElements();
     698             :     assert(NumElts == NVT.getVectorNumElements() &&
     699             :            "Dst and Src must have the same number of elements");
     700             :     assert(isPowerOf2_32(NumElts) &&
     701             :            "Promoted vector type must be a power of two");
     702             : 
     703          20 :     SDValue EOp1, EOp2;
     704          20 :     GetSplitVector(InOp, EOp1, EOp2);
     705             : 
     706          20 :     EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
     707          40 :                                    NumElts/2);
     708          40 :     EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
     709          40 :     EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
     710             : 
     711          40 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
     712             :   }
     713           1 :   case TargetLowering::TypeWidenVector: {
     714           1 :     SDValue WideInOp = GetWidenedVector(InOp);
     715             : 
     716             :     // Truncate widened InOp.
     717           2 :     unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
     718           1 :     EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
     719           3 :                                    N->getValueType(0).getScalarType(), NumElem);
     720           2 :     SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
     721             : 
     722             :     // Zero extend so that the elements are of same type as those of NVT
     723           1 :     EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
     724           2 :                                  NumElem);
     725           2 :     SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
     726             : 
     727             :     // Extract the low NVT subvector.
     728           2 :     MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
     729           2 :     SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
     730           2 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
     731             :   }
     732             :   }
     733             : 
     734             :   // Truncate to NVT instead of VT
     735       70704 :   return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
     736             : }
     737             : 
     738         483 : SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
     739         483 :   if (ResNo == 1)
     740         478 :     return PromoteIntRes_Overflow(N);
     741             : 
     742             :   // The operation overflowed iff the result in the larger type is not the
     743             :   // zero extension of its truncation to the original type.
     744          10 :   SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
     745          10 :   SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
     746          10 :   EVT OVT = N->getOperand(0).getValueType();
     747          10 :   EVT NVT = LHS.getValueType();
     748             :   SDLoc dl(N);
     749             : 
     750             :   // Do the arithmetic in the larger type.
     751           5 :   unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
     752          10 :   SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
     753             : 
     754             :   // Calculate the overflow flag: zero extend the arithmetic result from
     755             :   // the original type.
     756           5 :   SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
     757             :   // Overflowed if and only if this is not equal to Res.
     758          10 :   Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
     759             : 
     760             :   // Use the calculated overflow everywhere.
     761           5 :   ReplaceValueWith(SDValue(N, 1), Ofl);
     762             : 
     763           5 :   return Res;
     764             : }
     765             : 
     766          15 : SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
     767          15 :   if (ResNo == 1)
     768          10 :     return PromoteIntRes_Overflow(N);
     769             : 
     770             :   // We need to sign-extend the operands so the carry value computed by the
     771             :   // wide operation will be equivalent to the carry value computed by the
     772             :   // narrow operation.
     773             :   // An ADDCARRY can generate carry only if any of the operands has its
     774             :   // most significant bit set. Sign extension propagates the most significant
     775             :   // bit into the higher bits which means the extra bit that the narrow
     776             :   // addition would need (i.e. the carry) will be propagated through the higher
     777             :   // bits of the wide addition.
     778             :   // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
     779             :   // preserved by sign extension.
     780          10 :   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
     781          10 :   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
     782             : 
     783          15 :   EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
     784             : 
     785             :   // Do the arithmetic in the wide type.
     786          10 :   SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
     787          25 :                             LHS, RHS, N->getOperand(2));
     788             : 
     789             :   // Update the users of the original carry/borrow value.
     790           5 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     791             : 
     792           5 :   return SDValue(Res.getNode(), 0);
     793             : }
     794             : 
     795         102 : SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
     796             :   // Promote the overflow bit trivially.
     797         102 :   if (ResNo == 1)
     798         101 :     return PromoteIntRes_Overflow(N);
     799             : 
     800           1 :   SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
     801             :   SDLoc DL(N);
     802           1 :   EVT SmallVT = LHS.getValueType();
     803             : 
     804             :   // To determine if the result overflowed in a larger type, we extend the
     805             :   // input to the larger type, do the multiply (checking if it overflows),
     806             :   // then also check the high bits of the result to see if overflow happened
     807             :   // there.
     808           1 :   if (N->getOpcode() == ISD::SMULO) {
     809           1 :     LHS = SExtPromotedInteger(LHS);
     810           1 :     RHS = SExtPromotedInteger(RHS);
     811             :   } else {
     812           0 :     LHS = ZExtPromotedInteger(LHS);
     813           0 :     RHS = ZExtPromotedInteger(RHS);
     814             :   }
     815           2 :   SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
     816           2 :   SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
     817             : 
     818             :   // Overflow occurred if it occurred in the larger type, or if the high part
     819             :   // of the result does not zero/sign-extend the low part.  Check this second
     820             :   // possibility first.
     821           1 :   SDValue Overflow;
     822           1 :   if (N->getOpcode() == ISD::UMULO) {
     823             :     // Unsigned overflow occurred if the high part is non-zero.
     824           0 :     SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
     825           0 :                              DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
     826           0 :                                                    DL));
     827           0 :     Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
     828             :                             DAG.getConstant(0, DL, Hi.getValueType()),
     829           0 :                             ISD::SETNE);
     830             :   } else {
     831             :     // Signed overflow occurred if the high part does not sign extend the low.
     832           1 :     SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
     833           2 :                                Mul, DAG.getValueType(SmallVT));
     834           2 :     Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
     835             :   }
     836             : 
     837             :   // The only other way for overflow to occur is if the multiplication in the
     838             :   // larger type itself overflowed.
     839           2 :   Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
     840           2 :                          SDValue(Mul.getNode(), 1));
     841             : 
     842             :   // Use the calculated overflow everywhere.
     843           1 :   ReplaceValueWith(SDValue(N, 1), Overflow);
     844           1 :   return Mul;
     845             : }
     846             : 
     847        2355 : SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
     848        4710 :   return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
     849        4710 :                                                N->getValueType(0)));
     850             : }
     851             : 
     852          37 : SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
     853          37 :   SDValue Chain = N->getOperand(0); // Get the chain.
     854          37 :   SDValue Ptr = N->getOperand(1); // Get the pointer.
     855          74 :   EVT VT = N->getValueType(0);
     856             :   SDLoc dl(N);
     857             : 
     858          37 :   MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
     859          37 :   unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
     860             :   // The argument is passed as NumRegs registers of type RegVT.
     861             : 
     862          74 :   SmallVector<SDValue, 8> Parts(NumRegs);
     863         111 :   for (unsigned i = 0; i < NumRegs; ++i) {
     864         111 :     Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
     865          74 :                             N->getConstantOperandVal(3));
     866          37 :     Chain = Parts[i].getValue(1);
     867             :   }
     868             : 
     869             :   // Handle endianness of the load.
     870          74 :   if (DAG.getDataLayout().isBigEndian())
     871             :     std::reverse(Parts.begin(), Parts.end());
     872             : 
     873             :   // Assemble the parts in the promoted type.
     874         111 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     875          74 :   SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
     876          37 :   for (unsigned i = 1; i < NumRegs; ++i) {
     877           0 :     SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
     878             :     // Shift it to the right position and "or" it in.
     879           0 :     Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
     880           0 :                        DAG.getConstant(i * RegVT.getSizeInBits(), dl,
     881           0 :                                        TLI.getPointerTy(DAG.getDataLayout())));
     882           0 :     Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
     883             :   }
     884             : 
     885             :   // Modified the chain result - switch anything that used the old chain to
     886             :   // use the new one.
     887          37 :   ReplaceValueWith(SDValue(N, 1), Chain);
     888             : 
     889          74 :   return Res;
     890             : }
     891             : 
     892             : //===----------------------------------------------------------------------===//
     893             : //  Integer Operand Promotion
     894             : //===----------------------------------------------------------------------===//
     895             : 
     896             : /// PromoteIntegerOperand - This method is called when the specified operand of
     897             : /// the specified node is found to need promotion.  At this point, all of the
     898             : /// result types of the node are known to be legal, but other operands of the
     899             : /// node may need promotion or expansion as well as the specified one.
     900      165596 : bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
     901             :   LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
     902             :              dbgs() << "\n");
     903      165596 :   SDValue Res = SDValue();
     904             : 
     905      496788 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
     906             :     LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
     907             :     return false;
     908             :   }
     909             : 
     910      329042 :   switch (N->getOpcode()) {
     911           0 :     default:
     912             :   #ifndef NDEBUG
     913             :     dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
     914             :     N->dump(&DAG); dbgs() << "\n";
     915             :   #endif
     916           0 :     llvm_unreachable("Do not know how to promote this operator's operand!");
     917             : 
     918       10723 :   case ISD::ANY_EXTEND:   Res = PromoteIntOp_ANY_EXTEND(N); break;
     919             :   case ISD::ATOMIC_STORE:
     920         111 :     Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
     921         111 :     break;
     922         407 :   case ISD::BITCAST:      Res = PromoteIntOp_BITCAST(N); break;
     923          12 :   case ISD::BR_CC:        Res = PromoteIntOp_BR_CC(N, OpNo); break;
     924       38143 :   case ISD::BRCOND:       Res = PromoteIntOp_BRCOND(N, OpNo); break;
     925           0 :   case ISD::BUILD_PAIR:   Res = PromoteIntOp_BUILD_PAIR(N); break;
     926        3953 :   case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
     927        2686 :   case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
     928        5133 :   case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
     929         290 :   case ISD::INSERT_VECTOR_ELT:
     930         290 :                           Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
     931          41 :   case ISD::SCALAR_TO_VECTOR:
     932          41 :                           Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
     933        3664 :   case ISD::VSELECT:
     934        3664 :   case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
     935         106 :   case ISD::SELECT_CC:    Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
     936        2334 :   case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
     937        7429 :   case ISD::SIGN_EXTEND:  Res = PromoteIntOp_SIGN_EXTEND(N); break;
     938         133 :   case ISD::SINT_TO_FP:   Res = PromoteIntOp_SINT_TO_FP(N); break;
     939        6645 :   case ISD::STORE:        Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
     940        6645 :                                                    OpNo); break;
     941          24 :   case ISD::MSTORE:       Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
     942          24 :                                                     OpNo); break;
     943          38 :   case ISD::MLOAD:        Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
     944          38 :                                                     OpNo); break;
     945          45 :   case ISD::MGATHER:  Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
     946          45 :                                                  OpNo); break;
     947           6 :   case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
     948           6 :                                                   OpNo); break;
     949        3294 :   case ISD::TRUNCATE:     Res = PromoteIntOp_TRUNCATE(N); break;
     950        2436 :   case ISD::FP16_TO_FP:
     951        2436 :   case ISD::UINT_TO_FP:   Res = PromoteIntOp_UINT_TO_FP(N); break;
     952       76858 :   case ISD::ZERO_EXTEND:  Res = PromoteIntOp_ZERO_EXTEND(N); break;
     953           0 :   case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
     954             : 
     955           0 :   case ISD::SHL:
     956             :   case ISD::SRA:
     957             :   case ISD::SRL:
     958             :   case ISD::ROTL:
     959           0 :   case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
     960             : 
     961          10 :   case ISD::ADDCARRY:
     962          10 :   case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
     963             :   }
     964             : 
     965             :   // If the result is null, the sub-method took care of registering results etc.
     966      164521 :   if (!Res.getNode()) return false;
     967             : 
     968             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
     969             :   // core about this.
     970      164521 :   if (Res.getNode() == N)
     971             :     return true;
     972             : 
     973             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
     974             :          "Invalid operand expansion");
     975             : 
     976      113783 :   ReplaceValueWith(SDValue(N, 0), Res);
     977      113783 :   return false;
     978             : }
     979             : 
     980             : /// PromoteSetCCOperands - Promote the operands of a comparison.  This code is
     981             : /// shared among BR_CC, SELECT_CC, and SETCC handlers.
     982        2452 : void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
     983             :                                             ISD::CondCode CCCode) {
     984             :   // We have to insert explicit sign or zero extends.  Note that we could
     985             :   // insert sign extends for ALL conditions, but zero extend is cheaper on
     986             :   // many machines (an AND instead of two shifts), so prefer it.
     987             :   switch (CCCode) {
     988           0 :   default: llvm_unreachable("Unknown integer comparison!");
     989        1691 :   case ISD::SETEQ:
     990             :   case ISD::SETNE: {
     991        1691 :     SDValue OpL = GetPromotedInteger(NewLHS);
     992        1691 :     SDValue OpR = GetPromotedInteger(NewRHS);
     993             : 
     994             :     // We would prefer to promote the comparison operand with sign extension.
     995             :     // If the width of OpL/OpR excluding the duplicated sign bits is no greater
     996             :     // than the width of NewLHS/NewRH, we can avoid inserting real truncate
     997             :     // instruction, which is redudant eventually.
     998             :     unsigned OpLEffectiveBits =
     999        1691 :         OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
    1000             :     unsigned OpREffectiveBits =
    1001        1691 :         OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
    1002        1889 :     if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
    1003         198 :         OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
    1004         196 :       NewLHS = OpL;
    1005         196 :       NewRHS = OpR;
    1006             :     } else {
    1007        1495 :       NewLHS = ZExtPromotedInteger(NewLHS);
    1008        1495 :       NewRHS = ZExtPromotedInteger(NewRHS);
    1009             :     }
    1010             :     break;
    1011             :   }
    1012         317 :   case ISD::SETUGE:
    1013             :   case ISD::SETUGT:
    1014             :   case ISD::SETULE:
    1015             :   case ISD::SETULT:
    1016             :     // ALL of these operations will work if we either sign or zero extend
    1017             :     // the operands (including the unsigned comparisons!).  Zero extend is
    1018             :     // usually a simpler/cheaper operation, so prefer it.
    1019         317 :     NewLHS = ZExtPromotedInteger(NewLHS);
    1020         317 :     NewRHS = ZExtPromotedInteger(NewRHS);
    1021         317 :     break;
    1022         444 :   case ISD::SETGE:
    1023             :   case ISD::SETGT:
    1024             :   case ISD::SETLT:
    1025             :   case ISD::SETLE:
    1026         444 :     NewLHS = SExtPromotedInteger(NewLHS);
    1027         444 :     NewRHS = SExtPromotedInteger(NewRHS);
    1028         444 :     break;
    1029             :   }
    1030        2452 : }
    1031             : 
    1032       10723 : SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
    1033       21446 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1034       42892 :   return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
    1035             : }
    1036             : 
    1037         111 : SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
    1038         222 :   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
    1039         222 :   return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
    1040         444 :                        N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
    1041             : }
    1042             : 
    1043           0 : SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
    1044             :   // This should only occur in unusual situations like bitcasting to an
    1045             :   // x86_fp80, so just turn it into a store+load
    1046        1221 :   return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
    1047             : }
    1048             : 
    1049          12 : SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
    1050             :   assert(OpNo == 2 && "Don't know how to promote this operand!");
    1051             : 
    1052          12 :   SDValue LHS = N->getOperand(2);
    1053          12 :   SDValue RHS = N->getOperand(3);
    1054          12 :   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
    1055             : 
    1056             :   // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
    1057             :   // legal types.
    1058          24 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1059          12 :                                 N->getOperand(1), LHS, RHS, N->getOperand(4)),
    1060          12 :                  0);
    1061             : }
    1062             : 
    1063       38143 : SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
    1064             :   assert(OpNo == 1 && "only know how to promote condition");
    1065             : 
    1066             :   // Promote all the way up to the canonical SetCC type.
    1067       76286 :   SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
    1068             : 
    1069             :   // The chain (Op#0) and basic block destination (Op#2) are always legal types.
    1070       76286 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
    1071       76286 :                                         N->getOperand(2)), 0);
    1072             : }
    1073             : 
    1074           0 : SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
    1075             :   // Since the result type is legal, the operands must promote to it.
    1076           0 :   EVT OVT = N->getOperand(0).getValueType();
    1077           0 :   SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
    1078           0 :   SDValue Hi = GetPromotedInteger(N->getOperand(1));
    1079             :   assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
    1080             :   SDLoc dl(N);
    1081             : 
    1082           0 :   Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
    1083           0 :                    DAG.getConstant(OVT.getSizeInBits(), dl,
    1084           0 :                                    TLI.getPointerTy(DAG.getDataLayout())));
    1085           0 :   return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
    1086             : }
    1087             : 
    1088        3953 : SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
    1089             :   // The vector type is legal but the element type is not.  This implies
    1090             :   // that the vector is a power-of-two in length and that the element
    1091             :   // type does not have a strange size (eg: it is not i1).
    1092        7906 :   EVT VecVT = N->getValueType(0);
    1093        3953 :   unsigned NumElts = VecVT.getVectorNumElements();
    1094             :   assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
    1095             :          "Legal vector of one illegal element?");
    1096             : 
    1097             :   // Promote the inserted value.  The type does not need to match the
    1098             :   // vector element type.  Check that any extra bits introduced will be
    1099             :   // truncated away.
    1100             :   assert(N->getOperand(0).getValueSizeInBits() >=
    1101             :          N->getValueType(0).getScalarSizeInBits() &&
    1102             :          "Type of inserted value narrower than vector element type!");
    1103             : 
    1104             :   SmallVector<SDValue, 16> NewOps;
    1105       77331 :   for (unsigned i = 0; i < NumElts; ++i)
    1106       73378 :     NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
    1107             : 
    1108       11859 :   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1109             : }
    1110             : 
    1111         290 : SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
    1112             :                                                          unsigned OpNo) {
    1113         290 :   if (OpNo == 1) {
    1114             :     // Promote the inserted value.  This is valid because the type does not
    1115             :     // have to match the vector element type.
    1116             : 
    1117             :     // Check that any extra bits introduced will be truncated away.
    1118             :     assert(N->getOperand(1).getValueSizeInBits() >=
    1119             :            N->getValueType(0).getScalarSizeInBits() &&
    1120             :            "Type of inserted value narrower than vector element type!");
    1121         870 :     return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1122             :                                   GetPromotedInteger(N->getOperand(1)),
    1123         290 :                                   N->getOperand(2)),
    1124         290 :                    0);
    1125             :   }
    1126             : 
    1127             :   assert(OpNo == 2 && "Different operand and result vector types?");
    1128             : 
    1129             :   // Promote the index.
    1130           0 :   SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
    1131           0 :                                    TLI.getVectorIdxTy(DAG.getDataLayout()));
    1132           0 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1133           0 :                                 N->getOperand(1), Idx), 0);
    1134             : }
    1135             : 
    1136          41 : SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
    1137             :   // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
    1138             :   // the operand in place.
    1139          82 :   return SDValue(DAG.UpdateNodeOperands(N,
    1140          82 :                                 GetPromotedInteger(N->getOperand(0))), 0);
    1141             : }
    1142             : 
    1143        3664 : SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
    1144             :   assert(OpNo == 0 && "Only know how to promote the condition!");
    1145        3664 :   SDValue Cond = N->getOperand(0);
    1146        7328 :   EVT OpTy = N->getOperand(1).getValueType();
    1147             : 
    1148        3664 :   if (N->getOpcode() == ISD::VSELECT)
    1149         560 :     if (SDValue Res = WidenVSELECTAndMask(N))
    1150         479 :       return Res;
    1151             : 
    1152             :   // Promote all the way up to the canonical SetCC type.
    1153        3185 :   EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
    1154        3185 :   Cond = PromoteTargetBoolean(Cond, OpVT);
    1155             : 
    1156        6370 :   return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
    1157        6370 :                                         N->getOperand(2)), 0);
    1158             : }
    1159             : 
    1160         106 : SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
    1161             :   assert(OpNo == 0 && "Don't know how to promote this operand!");
    1162             : 
    1163         106 :   SDValue LHS = N->getOperand(0);
    1164         106 :   SDValue RHS = N->getOperand(1);
    1165         106 :   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
    1166             : 
    1167             :   // The CC (#4) and the possible return values (#2 and #3) have legal types.
    1168         212 :   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
    1169         212 :                                 N->getOperand(3), N->getOperand(4)), 0);
    1170             : }
    1171             : 
    1172        2334 : SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
    1173             :   assert(OpNo == 0 && "Don't know how to promote this operand!");
    1174             : 
    1175        2334 :   SDValue LHS = N->getOperand(0);
    1176        2334 :   SDValue RHS = N->getOperand(1);
    1177        2334 :   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
    1178             : 
    1179             :   // The CC (#2) is always legal.
    1180        4668 :   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
    1181             : }
    1182             : 
    1183           0 : SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
    1184           0 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1185           0 :                                 ZExtPromotedInteger(N->getOperand(1))), 0);
    1186             : }
    1187             : 
    1188        7429 : SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
    1189       14858 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1190             :   SDLoc dl(N);
    1191       22287 :   Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
    1192        7429 :   return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
    1193       29716 :                      Op, DAG.getValueType(N->getOperand(0).getValueType()));
    1194             : }
    1195             : 
    1196         133 : SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
    1197         266 :   return SDValue(DAG.UpdateNodeOperands(N,
    1198         266 :                                 SExtPromotedInteger(N->getOperand(0))), 0);
    1199             : }
    1200             : 
    1201        6645 : SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
    1202             :   assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
    1203        6645 :   SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
    1204             :   SDLoc dl(N);
    1205             : 
    1206        6645 :   SDValue Val = GetPromotedInteger(N->getValue());  // Get promoted value.
    1207             : 
    1208             :   // Truncate the value and store the result.
    1209        6645 :   return DAG.getTruncStore(Ch, dl, Val, Ptr,
    1210       13290 :                            N->getMemoryVT(), N->getMemOperand());
    1211             : }
    1212             : 
    1213          26 : SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
    1214             :                                               unsigned OpNo) {
    1215             : 
    1216          26 :   SDValue DataOp = N->getValue();
    1217          26 :   EVT DataVT = DataOp.getValueType();
    1218          26 :   SDValue Mask = N->getMask();
    1219             :   SDLoc dl(N);
    1220             : 
    1221             :   bool TruncateStore = false;
    1222          26 :   if (OpNo == 2) {
    1223             :     // Mask comes before the data operand. If the data operand is legal, we just
    1224             :     // promote the mask.
    1225             :     // When the data operand has illegal type, we should legalize the data
    1226             :     // operand first. The mask will be promoted/splitted/widened according to
    1227             :     // the data operand type.
    1228          22 :     if (TLI.isTypeLegal(DataVT)) {
    1229          16 :       Mask = PromoteTargetBoolean(Mask, DataVT);
    1230             :       // Update in place.
    1231          16 :       SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
    1232          16 :       NewOps[2] = Mask;
    1233          32 :       return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1234             :     }
    1235             : 
    1236          12 :     if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
    1237           2 :       return PromoteIntOp_MSTORE(N, 3);
    1238           8 :     if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector)
    1239           2 :       return WidenVecOp_MSTORE(N, 3);
    1240             :     assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
    1241           2 :     return SplitVecOp_MSTORE(N, 3);
    1242             :   } else { // Data operand
    1243             :     assert(OpNo == 3 && "Unexpected operand for promotion");
    1244           4 :     DataOp = GetPromotedInteger(DataOp);
    1245             :     TruncateStore = true;
    1246             :   }
    1247             : 
    1248           4 :   return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
    1249             :                             N->getMemoryVT(), N->getMemOperand(),
    1250           8 :                             TruncateStore, N->isCompressingStore());
    1251             : }
    1252             : 
    1253          38 : SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
    1254             :                                              unsigned OpNo) {
    1255             :   assert(OpNo == 2 && "Only know how to promote the mask!");
    1256          76 :   EVT DataVT = N->getValueType(0);
    1257          76 :   SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
    1258          38 :   SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
    1259          38 :   NewOps[OpNo] = Mask;
    1260         114 :   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1261             : }
    1262             : 
    1263          45 : SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
    1264             :                                                unsigned OpNo) {
    1265             : 
    1266          45 :   SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
    1267          45 :   if (OpNo == 2) {
    1268             :     // The Mask
    1269          56 :     EVT DataVT = N->getValueType(0);
    1270          84 :     NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
    1271          17 :   } else if (OpNo == 4) {
    1272             :     // Need to sign extend the index since the bits will likely be used.
    1273          51 :     NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
    1274             :   } else
    1275           0 :     NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
    1276             : 
    1277          90 :   SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1278             :   // updated in place.
    1279          45 :   if (Res.getNode() == N)
    1280          45 :     return Res;
    1281             : 
    1282           0 :   ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
    1283           0 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    1284           0 :   return SDValue();
    1285             : }
    1286             : 
    1287           6 : SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
    1288             :                                                 unsigned OpNo) {
    1289           6 :   SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
    1290           6 :   if (OpNo == 2) {
    1291             :     // The Mask
    1292           0 :     EVT DataVT = N->getValue().getValueType();
    1293           0 :     NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
    1294           6 :   } else if (OpNo == 4) {
    1295             :     // Need to sign extend the index since the bits will likely be used.
    1296          18 :     NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
    1297             :   } else
    1298           0 :     NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
    1299          18 :   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1300             : }
    1301             : 
    1302        3294 : SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
    1303        6588 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1304       13176 :   return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
    1305             : }
    1306             : 
    1307        2436 : SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
    1308        4872 :   return SDValue(DAG.UpdateNodeOperands(N,
    1309        4872 :                                 ZExtPromotedInteger(N->getOperand(0))), 0);
    1310             : }
    1311             : 
    1312       76858 : SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
    1313             :   SDLoc dl(N);
    1314      153716 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1315      230574 :   Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
    1316       76858 :   return DAG.getZeroExtendInReg(Op, dl,
    1317      230574 :                                 N->getOperand(0).getValueType().getScalarType());
    1318             : }
    1319             : 
    1320          10 : SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
    1321             :   assert(OpNo == 2 && "Don't know how to promote this operand!");
    1322             : 
    1323          10 :   SDValue LHS = N->getOperand(0);
    1324          10 :   SDValue RHS = N->getOperand(1);
    1325          10 :   SDValue Carry = N->getOperand(2);
    1326             :   SDLoc DL(N);
    1327             : 
    1328          10 :   auto VT = getSetCCResultType(LHS.getValueType());
    1329          10 :   TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT);
    1330          10 :   switch (BoolType) {
    1331           0 :   case TargetLoweringBase::UndefinedBooleanContent:
    1332           0 :     Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
    1333           0 :     break;
    1334          10 :   case TargetLoweringBase::ZeroOrOneBooleanContent:
    1335          10 :     Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
    1336          10 :     break;
    1337           0 :   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
    1338           0 :     Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
    1339           0 :     break;
    1340             :   }
    1341             : 
    1342          20 :   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
    1343             : }
    1344             : 
    1345             : //===----------------------------------------------------------------------===//
    1346             : //  Integer Result Expansion
    1347             : //===----------------------------------------------------------------------===//
    1348             : 
    1349             : /// ExpandIntegerResult - This method is called when the specified result of the
    1350             : /// specified node is found to need expansion.  At this point, the node may also
    1351             : /// have invalid operands or may have other results that need promotion, we just
    1352             : /// know that (at least) one result needs expansion.
    1353      353570 : void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
    1354             :   LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
    1355             :              dbgs() << "\n");
    1356             :   SDValue Lo, Hi;
    1357      353570 :   Lo = Hi = SDValue();
    1358             : 
    1359             :   // See if the target wants to custom expand this node.
    1360      707140 :   if (CustomLowerNode(N, N->getValueType(ResNo), true))
    1361         783 :     return;
    1362             : 
    1363      705574 :   switch (N->getOpcode()) {
    1364           0 :   default:
    1365             : #ifndef NDEBUG
    1366             :     dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
    1367             :     N->dump(&DAG); dbgs() << "\n";
    1368             : #endif
    1369           0 :     llvm_unreachable("Do not know how to expand the result of this operator!");
    1370             : 
    1371          53 :   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
    1372         542 :   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
    1373         623 :   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
    1374        1846 :   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
    1375             : 
    1376       10604 :   case ISD::BITCAST:            ExpandRes_BITCAST(N, Lo, Hi); break;
    1377       12129 :   case ISD::BUILD_PAIR:         ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
    1378        2740 :   case ISD::EXTRACT_ELEMENT:    ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
    1379        1840 :   case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
    1380          15 :   case ISD::VAARG:              ExpandRes_VAARG(N, Lo, Hi); break;
    1381             : 
    1382        3661 :   case ISD::ANY_EXTEND:  ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
    1383         317 :   case ISD::AssertSext:  ExpandIntRes_AssertSext(N, Lo, Hi); break;
    1384         315 :   case ISD::AssertZext:  ExpandIntRes_AssertZext(N, Lo, Hi); break;
    1385           1 :   case ISD::BITREVERSE:  ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
    1386          44 :   case ISD::BSWAP:       ExpandIntRes_BSWAP(N, Lo, Hi); break;
    1387       68919 :   case ISD::Constant:    ExpandIntRes_Constant(N, Lo, Hi); break;
    1388          60 :   case ISD::CTLZ_ZERO_UNDEF:
    1389          60 :   case ISD::CTLZ:        ExpandIntRes_CTLZ(N, Lo, Hi); break;
    1390          12 :   case ISD::CTPOP:       ExpandIntRes_CTPOP(N, Lo, Hi); break;
    1391          14 :   case ISD::CTTZ_ZERO_UNDEF:
    1392          14 :   case ISD::CTTZ:        ExpandIntRes_CTTZ(N, Lo, Hi); break;
    1393           1 :   case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
    1394          93 :   case ISD::FP_TO_SINT:  ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
    1395          86 :   case ISD::FP_TO_UINT:  ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
    1396       71164 :   case ISD::LOAD:        ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
    1397        1762 :   case ISD::MUL:         ExpandIntRes_MUL(N, Lo, Hi); break;
    1398           2 :   case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
    1399          69 :   case ISD::SDIV:        ExpandIntRes_SDIV(N, Lo, Hi); break;
    1400        1678 :   case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
    1401         159 :   case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
    1402          74 :   case ISD::SREM:        ExpandIntRes_SREM(N, Lo, Hi); break;
    1403       13613 :   case ISD::TRUNCATE:    ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
    1404          68 :   case ISD::UDIV:        ExpandIntRes_UDIV(N, Lo, Hi); break;
    1405          75 :   case ISD::UREM:        ExpandIntRes_UREM(N, Lo, Hi); break;
    1406        5840 :   case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
    1407          13 :   case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
    1408             : 
    1409          49 :   case ISD::ATOMIC_LOAD_ADD:
    1410             :   case ISD::ATOMIC_LOAD_SUB:
    1411             :   case ISD::ATOMIC_LOAD_AND:
    1412             :   case ISD::ATOMIC_LOAD_CLR:
    1413             :   case ISD::ATOMIC_LOAD_OR:
    1414             :   case ISD::ATOMIC_LOAD_XOR:
    1415             :   case ISD::ATOMIC_LOAD_NAND:
    1416             :   case ISD::ATOMIC_LOAD_MIN:
    1417             :   case ISD::ATOMIC_LOAD_MAX:
    1418             :   case ISD::ATOMIC_LOAD_UMIN:
    1419             :   case ISD::ATOMIC_LOAD_UMAX:
    1420             :   case ISD::ATOMIC_SWAP:
    1421             :   case ISD::ATOMIC_CMP_SWAP: {
    1422          49 :     std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
    1423          49 :     SplitInteger(Tmp.first, Lo, Hi);
    1424          49 :     ReplaceValueWith(SDValue(N, 1), Tmp.second);
    1425             :     break;
    1426             :   }
    1427             :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
    1428             :     AtomicSDNode *AN = cast<AtomicSDNode>(N);
    1429         129 :     SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
    1430          43 :     SDValue Tmp = DAG.getAtomicCmpSwap(
    1431          43 :         ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
    1432          43 :         N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
    1433          86 :         AN->getMemOperand());
    1434             : 
    1435             :     // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
    1436             :     // success simply by comparing the loaded value against the ingoing
    1437             :     // comparison.
    1438          86 :     SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
    1439         129 :                                    N->getOperand(2), ISD::SETEQ);
    1440             : 
    1441          43 :     SplitInteger(Tmp, Lo, Hi);
    1442          43 :     ReplaceValueWith(SDValue(N, 1), Success);
    1443          86 :     ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
    1444             :     break;
    1445             :   }
    1446             : 
    1447       17125 :   case ISD::AND:
    1448             :   case ISD::OR:
    1449       17125 :   case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
    1450             : 
    1451           9 :   case ISD::UMAX:
    1452             :   case ISD::SMAX:
    1453             :   case ISD::UMIN:
    1454           9 :   case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
    1455             : 
    1456       69788 :   case ISD::ADD:
    1457       69788 :   case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
    1458             : 
    1459           8 :   case ISD::ADDC:
    1460           8 :   case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
    1461             : 
    1462           8 :   case ISD::ADDE:
    1463           8 :   case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
    1464             : 
    1465         786 :   case ISD::ADDCARRY:
    1466         786 :   case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
    1467             : 
    1468       65941 :   case ISD::SHL:
    1469             :   case ISD::SRA:
    1470       65941 :   case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
    1471             : 
    1472          20 :   case ISD::SADDO:
    1473          20 :   case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
    1474         566 :   case ISD::UADDO:
    1475         566 :   case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
    1476          12 :   case ISD::UMULO:
    1477          12 :   case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
    1478             :   }
    1479             : 
    1480             :   // If Lo/Hi is null, the sub-method took care of registering results etc.
    1481      352787 :   if (Lo.getNode())
    1482      352774 :     SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
    1483             : }
    1484             : 
    1485             : /// Lower an atomic node to the appropriate builtin call.
    1486          49 : std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
    1487          49 :   unsigned Opc = Node->getOpcode();
    1488          49 :   MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
    1489          49 :   RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
    1490             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
    1491             : 
    1492          49 :   return ExpandChainLibCall(LC, Node, false);
    1493             : }
    1494             : 
    1495             : /// N is a shift by a value that needs to be expanded,
    1496             : /// and the shift amount is a constant 'Amt'.  Expand the operation.
    1497       65191 : void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
    1498             :                                              SDValue &Lo, SDValue &Hi) {
    1499             :   SDLoc DL(N);
    1500             :   // Expand the incoming operand to be shifted, so that we have its parts
    1501       65191 :   SDValue InL, InH;
    1502      130382 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    1503             : 
    1504             :   // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
    1505             :   // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
    1506       65191 :   if (!Amt) {
    1507           0 :     Lo = InL;
    1508           0 :     Hi = InH;
    1509           0 :     return;
    1510             :   }
    1511             : 
    1512      130382 :   EVT NVT = InL.getValueType();
    1513      130382 :   unsigned VTBits = N->getValueType(0).getSizeInBits();
    1514       65191 :   unsigned NVTBits = NVT.getSizeInBits();
    1515      130382 :   EVT ShTy = N->getOperand(1).getValueType();
    1516             : 
    1517      130382 :   if (N->getOpcode() == ISD::SHL) {
    1518       14213 :     if (Amt.ugt(VTBits)) {
    1519           2 :       Lo = Hi = DAG.getConstant(0, DL, NVT);
    1520       14211 :     } else if (Amt.ugt(NVTBits)) {
    1521        3032 :       Lo = DAG.getConstant(0, DL, NVT);
    1522        6064 :       Hi = DAG.getNode(ISD::SHL, DL,
    1523        9096 :                        NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
    1524       11179 :     } else if (Amt == NVTBits) {
    1525       10400 :       Lo = DAG.getConstant(0, DL, NVT);
    1526       10400 :       Hi = InL;
    1527             :     } else {
    1528        1558 :       Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
    1529        1558 :       Hi = DAG.getNode(ISD::OR, DL, NVT,
    1530         779 :                        DAG.getNode(ISD::SHL, DL, NVT, InH,
    1531             :                                    DAG.getConstant(Amt, DL, ShTy)),
    1532             :                        DAG.getNode(ISD::SRL, DL, NVT, InL,
    1533        4674 :                                    DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
    1534             :     }
    1535             :     return;
    1536             :   }
    1537             : 
    1538       50978 :   if (N->getOpcode() == ISD::SRL) {
    1539       50901 :     if (Amt.ugt(VTBits)) {
    1540           2 :       Lo = Hi = DAG.getConstant(0, DL, NVT);
    1541       50899 :     } else if (Amt.ugt(NVTBits)) {
    1542         150 :       Lo = DAG.getNode(ISD::SRL, DL,
    1543         225 :                        NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
    1544          75 :       Hi = DAG.getConstant(0, DL, NVT);
    1545       50824 :     } else if (Amt == NVTBits) {
    1546       31263 :       Lo = InH;
    1547       31263 :       Hi = DAG.getConstant(0, DL, NVT);
    1548             :     } else {
    1549       39122 :       Lo = DAG.getNode(ISD::OR, DL, NVT,
    1550       19561 :                        DAG.getNode(ISD::SRL, DL, NVT, InL,
    1551             :                                    DAG.getConstant(Amt, DL, ShTy)),
    1552             :                        DAG.getNode(ISD::SHL, DL, NVT, InH,
    1553      117366 :                                    DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
    1554       39122 :       Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
    1555             :     }
    1556             :     return;
    1557             :   }
    1558             : 
    1559             :   assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
    1560          77 :   if (Amt.ugt(VTBits)) {
    1561           4 :     Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1562           4 :                           DAG.getConstant(NVTBits - 1, DL, ShTy));
    1563          75 :   } else if (Amt.ugt(NVTBits)) {
    1564         122 :     Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1565         183 :                      DAG.getConstant(Amt - NVTBits, DL, ShTy));
    1566         122 :     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1567         122 :                      DAG.getConstant(NVTBits - 1, DL, ShTy));
    1568          14 :   } else if (Amt == NVTBits) {
    1569           3 :     Lo = InH;
    1570           6 :     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1571           6 :                      DAG.getConstant(NVTBits - 1, DL, ShTy));
    1572             :   } else {
    1573          22 :     Lo = DAG.getNode(ISD::OR, DL, NVT,
    1574          11 :                      DAG.getNode(ISD::SRL, DL, NVT, InL,
    1575             :                                  DAG.getConstant(Amt, DL, ShTy)),
    1576             :                      DAG.getNode(ISD::SHL, DL, NVT, InH,
    1577          66 :                                  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
    1578          22 :     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
    1579             :   }
    1580             : }
    1581             : 
    1582             : /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
    1583             : /// this shift based on knowledge of the high bit of the shift amount.  If we
    1584             : /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
    1585             : /// shift amount.
    1586         750 : bool DAGTypeLegalizer::
    1587             : ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
    1588         750 :   SDValue Amt = N->getOperand(1);
    1589        2250 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    1590         750 :   EVT ShTy = Amt.getValueType();
    1591             :   unsigned ShBits = ShTy.getScalarSizeInBits();
    1592             :   unsigned NVTBits = NVT.getScalarSizeInBits();
    1593             :   assert(isPowerOf2_32(NVTBits) &&
    1594             :          "Expanded integer type size not a power of two!");
    1595             :   SDLoc dl(N);
    1596             : 
    1597         750 :   APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
    1598         750 :   KnownBits Known;
    1599        1500 :   DAG.computeKnownBits(N->getOperand(1), Known);
    1600             : 
    1601             :   // If we don't know anything about the high bits, exit.
    1602        3000 :   if (((Known.Zero|Known.One) & HighBitMask) == 0)
    1603             :     return false;
    1604             : 
    1605             :   // Get the incoming operand to be shifted.
    1606          38 :   SDValue InL, InH;
    1607          76 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    1608             : 
    1609             :   // If we know that any of the high bits of the shift amount are one, then we
    1610             :   // can do this as a couple of simple shifts.
    1611          38 :   if (Known.One.intersects(HighBitMask)) {
    1612             :     // Mask out the high bit, which we know is set.
    1613           4 :     Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
    1614           6 :                       DAG.getConstant(~HighBitMask, dl, ShTy));
    1615             : 
    1616           4 :     switch (N->getOpcode()) {
    1617           0 :     default: llvm_unreachable("Unknown shift");
    1618           0 :     case ISD::SHL:
    1619           0 :       Lo = DAG.getConstant(0, dl, NVT);              // Low part is zero.
    1620           0 :       Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
    1621           0 :       return true;
    1622           2 :     case ISD::SRL:
    1623           2 :       Hi = DAG.getConstant(0, dl, NVT);              // Hi part is zero.
    1624           4 :       Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
    1625           2 :       return true;
    1626           0 :     case ISD::SRA:
    1627           0 :       Hi = DAG.getNode(ISD::SRA, dl, NVT, InH,       // Sign extend high part.
    1628           0 :                        DAG.getConstant(NVTBits - 1, dl, ShTy));
    1629           0 :       Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
    1630           0 :       return true;
    1631             :     }
    1632             :   }
    1633             : 
    1634             :   // If we know that all of the high bits of the shift amount are zero, then we
    1635             :   // can do this as a couple of simple shifts.
    1636          36 :   if (HighBitMask.isSubsetOf(Known.Zero)) {
    1637             :     // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
    1638             :     // shift if x is zero.  We can use XOR here because x is known to be smaller
    1639             :     // than 32.
    1640          11 :     SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
    1641          11 :                                DAG.getConstant(NVTBits - 1, dl, ShTy));
    1642             : 
    1643             :     unsigned Op1, Op2;
    1644          22 :     switch (N->getOpcode()) {
    1645           0 :     default: llvm_unreachable("Unknown shift");
    1646             :     case ISD::SHL:  Op1 = ISD::SHL; Op2 = ISD::SRL; break;
    1647           6 :     case ISD::SRL:
    1648           6 :     case ISD::SRA:  Op1 = ISD::SRL; Op2 = ISD::SHL; break;
    1649             :     }
    1650             : 
    1651             :     // When shifting right the arithmetic for Lo and Hi is swapped.
    1652          11 :     if (N->getOpcode() != ISD::SHL)
    1653             :       std::swap(InL, InH);
    1654             : 
    1655             :     // Use a little trick to get the bits that move from Lo to Hi. First
    1656             :     // shift by one bit.
    1657          22 :     SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
    1658             :     // Then compute the remaining shift with amount-1.
    1659          22 :     SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
    1660             : 
    1661          33 :     Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
    1662          22 :     Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
    1663             : 
    1664          11 :     if (N->getOpcode() != ISD::SHL)
    1665             :       std::swap(Hi, Lo);
    1666             :     return true;
    1667             :   }
    1668             : 
    1669             :   return false;
    1670             : }
    1671             : 
    1672             : /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
    1673             : /// of any size.
    1674          85 : bool DAGTypeLegalizer::
    1675             : ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
    1676          85 :   SDValue Amt = N->getOperand(1);
    1677         255 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    1678          85 :   EVT ShTy = Amt.getValueType();
    1679          85 :   unsigned NVTBits = NVT.getSizeInBits();
    1680             :   assert(isPowerOf2_32(NVTBits) &&
    1681             :          "Expanded integer type size not a power of two!");
    1682             :   SDLoc dl(N);
    1683             : 
    1684             :   // Get the incoming operand to be shifted.
    1685          85 :   SDValue InL, InH;
    1686         170 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    1687             : 
    1688          85 :   SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
    1689         170 :   SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
    1690         170 :   SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
    1691          85 :   SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
    1692          85 :                                  Amt, NVBitsNode, ISD::SETULT);
    1693          85 :   SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
    1694             :                                 Amt, DAG.getConstant(0, dl, ShTy),
    1695          85 :                                 ISD::SETEQ);
    1696             : 
    1697          85 :   SDValue LoS, HiS, LoL, HiL;
    1698         170 :   switch (N->getOpcode()) {
    1699           0 :   default: llvm_unreachable("Unknown shift");
    1700          40 :   case ISD::SHL:
    1701             :     // Short: ShAmt < NVTBits
    1702          80 :     LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
    1703          80 :     HiS = DAG.getNode(ISD::OR, dl, NVT,
    1704          40 :                       DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
    1705         120 :                       DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
    1706             : 
    1707             :     // Long: ShAmt >= NVTBits
    1708          40 :     LoL = DAG.getConstant(0, dl, NVT);                    // Lo part is zero.
    1709          80 :     HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
    1710             : 
    1711          40 :     Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
    1712          80 :     Hi = DAG.getSelect(dl, NVT, isZero, InH,
    1713          40 :                        DAG.getSelect(dl, NVT, isShort, HiS, HiL));
    1714          40 :     return true;
    1715          28 :   case ISD::SRL:
    1716             :     // Short: ShAmt < NVTBits
    1717          56 :     HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
    1718          56 :     LoS = DAG.getNode(ISD::OR, dl, NVT,
    1719          28 :                       DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
    1720             :     // FIXME: If Amt is zero, the following shift generates an undefined result
    1721             :     // on some architectures.
    1722          84 :                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
    1723             : 
    1724             :     // Long: ShAmt >= NVTBits
    1725          28 :     HiL = DAG.getConstant(0, dl, NVT);                    // Hi part is zero.
    1726          56 :     LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
    1727             : 
    1728          56 :     Lo = DAG.getSelect(dl, NVT, isZero, InL,
    1729          28 :                        DAG.getSelect(dl, NVT, isShort, LoS, LoL));
    1730          28 :     Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
    1731          28 :     return true;
    1732          17 :   case ISD::SRA:
    1733             :     // Short: ShAmt < NVTBits
    1734          34 :     HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
    1735          34 :     LoS = DAG.getNode(ISD::OR, dl, NVT,
    1736          17 :                       DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
    1737          51 :                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
    1738             : 
    1739             :     // Long: ShAmt >= NVTBits
    1740          34 :     HiL = DAG.getNode(ISD::SRA, dl, NVT, InH,             // Sign of Hi part.
    1741          34 :                       DAG.getConstant(NVTBits - 1, dl, ShTy));
    1742          34 :     LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
    1743             : 
    1744          34 :     Lo = DAG.getSelect(dl, NVT, isZero, InL,
    1745          17 :                        DAG.getSelect(dl, NVT, isShort, LoS, LoL));
    1746          17 :     Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
    1747          17 :     return true;
    1748             :   }
    1749             : }
    1750             : 
    1751           9 : static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
    1752             : 
    1753           9 :   switch (Op) {
    1754           0 :     default: llvm_unreachable("invalid min/max opcode");
    1755           2 :     case ISD::SMAX:
    1756           2 :       return std::make_pair(ISD::SETGT, ISD::UMAX);
    1757           3 :     case ISD::UMAX:
    1758           3 :       return std::make_pair(ISD::SETUGT, ISD::UMAX);
    1759           2 :     case ISD::SMIN:
    1760           2 :       return std::make_pair(ISD::SETLT, ISD::UMIN);
    1761           2 :     case ISD::UMIN:
    1762           2 :       return std::make_pair(ISD::SETULT, ISD::UMIN);
    1763             :   }
    1764             : }
    1765             : 
    1766           9 : void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
    1767             :                                            SDValue &Lo, SDValue &Hi) {
    1768             :   SDLoc DL(N);
    1769             :   ISD::NodeType LoOpc;
    1770             :   ISD::CondCode CondC;
    1771          27 :   std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
    1772             : 
    1773             :   // Expand the subcomponents.
    1774           9 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1775          18 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1776          18 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1777             : 
    1778             :   // Value types
    1779          18 :   EVT NVT = LHSL.getValueType();
    1780           9 :   EVT CCT = getSetCCResultType(NVT);
    1781             : 
    1782             :   // Hi part is always the same op
    1783          36 :   Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
    1784             : 
    1785             :   // We need to know whether to select Lo part that corresponds to 'winning'
    1786             :   // Hi part or if Hi parts are equal.
    1787           9 :   SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
    1788           9 :   SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
    1789             : 
    1790             :   // Lo part corresponding to the 'winning' Hi part
    1791           9 :   SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
    1792             : 
    1793             :   // Recursed Lo part if Hi parts are equal, this uses unsigned version
    1794          27 :   SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
    1795             : 
    1796           9 :   Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
    1797           9 : }
    1798             : 
    1799       69788 : void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
    1800             :                                            SDValue &Lo, SDValue &Hi) {
    1801             :   SDLoc dl(N);
    1802             :   // Expand the subcomponents.
    1803       69788 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1804      139576 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1805      139576 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1806             : 
    1807      139576 :   EVT NVT = LHSL.getValueType();
    1808       69788 :   SDValue LoOps[2] = { LHSL, RHSL };
    1809       69788 :   SDValue HiOps[3] = { LHSH, RHSH };
    1810             : 
    1811      209364 :   bool HasOpCarry = TLI.isOperationLegalOrCustom(
    1812       69788 :       N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
    1813      139576 :       TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
    1814       69788 :   if (HasOpCarry) {
    1815       68653 :     SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
    1816       68653 :     if (N->getOpcode() == ISD::ADD) {
    1817      136930 :       Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
    1818       68465 :       HiOps[2] = Lo.getValue(1);
    1819      136930 :       Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
    1820             :     } else {
    1821         376 :       Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
    1822         188 :       HiOps[2] = Lo.getValue(1);
    1823         376 :       Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
    1824             :     }
    1825             :     return;
    1826             :   }
    1827             : 
    1828             :   // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
    1829             :   // them.  TODO: Teach operation legalization how to expand unsupported
    1830             :   // ADDC/ADDE/SUBC/SUBE.  The problem is that these operations generate
    1831             :   // a carry of type MVT::Glue, but there doesn't seem to be any way to
    1832             :   // generate a value of this type in the expanded code sequence.
    1833             :   bool hasCarry =
    1834        2270 :     TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
    1835             :                                    ISD::ADDC : ISD::SUBC,
    1836        2270 :                                  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
    1837             : 
    1838        1135 :   if (hasCarry) {
    1839         238 :     SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
    1840         119 :     if (N->getOpcode() == ISD::ADD) {
    1841         202 :       Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
    1842         101 :       HiOps[2] = Lo.getValue(1);
    1843         202 :       Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
    1844             :     } else {
    1845          36 :       Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
    1846          18 :       HiOps[2] = Lo.getValue(1);
    1847          36 :       Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
    1848             :     }
    1849             :     return;
    1850             :   }
    1851             : 
    1852             :   bool hasOVF =
    1853        2032 :     TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
    1854             :                                    ISD::UADDO : ISD::USUBO,
    1855        2032 :                                  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
    1856        1016 :   TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
    1857             : 
    1858        1016 :   if (hasOVF) {
    1859         690 :     EVT OvfVT = getSetCCResultType(NVT);
    1860         690 :     SDVTList VTList = DAG.getVTList(NVT, OvfVT);
    1861             :     int RevOpc;
    1862         690 :     if (N->getOpcode() == ISD::ADD) {
    1863             :       RevOpc = ISD::SUB;
    1864         126 :       Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
    1865         126 :       Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
    1866             :     } else {
    1867             :       RevOpc = ISD::ADD;
    1868        1254 :       Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
    1869        1254 :       Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
    1870             :     }
    1871         690 :     SDValue OVF = Lo.getValue(1);
    1872             : 
    1873         690 :     switch (BoolType) {
    1874           0 :     case TargetLoweringBase::UndefinedBooleanContent:
    1875           0 :       OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
    1876             :       LLVM_FALLTHROUGH;
    1877           0 :     case TargetLoweringBase::ZeroOrOneBooleanContent:
    1878           0 :       OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
    1879           0 :       Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
    1880           0 :       break;
    1881         690 :     case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
    1882         690 :       OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
    1883        1380 :       Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
    1884             :     }
    1885             :     return;
    1886             :   }
    1887             : 
    1888         326 :   if (N->getOpcode() == ISD::ADD) {
    1889         462 :     Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
    1890         462 :     Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
    1891         231 :     SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
    1892         231 :                                 ISD::SETULT);
    1893             : 
    1894         231 :     if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
    1895         230 :       SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
    1896         460 :       Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
    1897             :       return;
    1898             :     }
    1899             : 
    1900           1 :     SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
    1901           1 :                                    DAG.getConstant(1, dl, NVT),
    1902           2 :                                    DAG.getConstant(0, dl, NVT));
    1903           1 :     SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
    1904           1 :                                 ISD::SETULT);
    1905           1 :     SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
    1906           1 :                                    DAG.getConstant(1, dl, NVT), Carry1);
    1907           2 :     Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
    1908             :   } else {
    1909         190 :     Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
    1910         190 :     Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
    1911             :     SDValue Cmp =
    1912          95 :       DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
    1913         190 :                    LoOps[0], LoOps[1], ISD::SETULT);
    1914             : 
    1915          95 :     SDValue Borrow;
    1916          95 :     if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
    1917          95 :       Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
    1918             :     else
    1919           0 :       Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
    1920           0 :                              DAG.getConstant(0, dl, NVT));
    1921             : 
    1922         190 :     Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
    1923             :   }
    1924             : }
    1925             : 
    1926           8 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
    1927             :                                             SDValue &Lo, SDValue &Hi) {
    1928             :   // Expand the subcomponents.
    1929           8 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1930             :   SDLoc dl(N);
    1931          16 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1932          16 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1933          24 :   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
    1934           8 :   SDValue LoOps[2] = { LHSL, RHSL };
    1935           8 :   SDValue HiOps[3] = { LHSH, RHSH };
    1936             : 
    1937           8 :   if (N->getOpcode() == ISD::ADDC) {
    1938          12 :     Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
    1939           6 :     HiOps[2] = Lo.getValue(1);
    1940          12 :     Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
    1941             :   } else {
    1942           4 :     Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
    1943           2 :     HiOps[2] = Lo.getValue(1);
    1944           4 :     Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
    1945             :   }
    1946             : 
    1947             :   // Legalized the flag result - switch anything that used the old flag to
    1948             :   // use the new one.
    1949          16 :   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
    1950           8 : }
    1951             : 
    1952           8 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
    1953             :                                             SDValue &Lo, SDValue &Hi) {
    1954             :   // Expand the subcomponents.
    1955           8 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1956             :   SDLoc dl(N);
    1957          16 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1958          16 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1959          24 :   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
    1960           8 :   SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
    1961           8 :   SDValue HiOps[3] = { LHSH, RHSH };
    1962             : 
    1963          24 :   Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
    1964           8 :   HiOps[2] = Lo.getValue(1);
    1965          24 :   Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
    1966             : 
    1967             :   // Legalized the flag result - switch anything that used the old flag to
    1968             :   // use the new one.
    1969          16 :   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
    1970           8 : }
    1971             : 
    1972         566 : void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
    1973             :                                              SDValue &Lo, SDValue &Hi) {
    1974         566 :   SDValue LHS = N->getOperand(0);
    1975         566 :   SDValue RHS = N->getOperand(1);
    1976             :   SDLoc dl(N);
    1977             : 
    1978         566 :   SDValue Ovf;
    1979             : 
    1980        1698 :   bool HasOpCarry = TLI.isOperationLegalOrCustom(
    1981         566 :       N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
    1982        1132 :       TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
    1983             : 
    1984         566 :   if (HasOpCarry) {
    1985             :     // Expand the subcomponents.
    1986         560 :     SDValue LHSL, LHSH, RHSL, RHSH;
    1987         560 :     GetExpandedInteger(LHS, LHSL, LHSH);
    1988         560 :     GetExpandedInteger(RHS, RHSL, RHSH);
    1989        1680 :     SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
    1990         560 :     SDValue LoOps[2] = { LHSL, RHSL };
    1991         560 :     SDValue HiOps[3] = { LHSH, RHSH };
    1992             : 
    1993        1120 :     unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
    1994        1120 :     Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
    1995         560 :     HiOps[2] = Lo.getValue(1);
    1996        1120 :     Hi = DAG.getNode(Opc, dl, VTList, HiOps);
    1997             : 
    1998         560 :     Ovf = Hi.getValue(1);
    1999             :   } else {
    2000             :     // Expand the result by simply replacing it with the equivalent
    2001             :     // non-overflow-checking operation.
    2002           6 :     auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
    2003          12 :     SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
    2004           6 :     SplitInteger(Sum, Lo, Hi);
    2005             : 
    2006             :     // Calculate the overflow: addition overflows iff a + b < a, and subtraction
    2007             :     // overflows iff a - b > a.
    2008           6 :     auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
    2009          12 :     Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
    2010             :   }
    2011             : 
    2012             :   // Legalized the flag result - switch anything that used the old flag to
    2013             :   // use the new one.
    2014         566 :   ReplaceValueWith(SDValue(N, 1), Ovf);
    2015         566 : }
    2016             : 
    2017         786 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
    2018             :                                                 SDValue &Lo, SDValue &Hi) {
    2019             :   // Expand the subcomponents.
    2020         786 :   SDValue LHSL, LHSH, RHSL, RHSH;
    2021             :   SDLoc dl(N);
    2022        1572 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    2023        1572 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    2024        2358 :   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
    2025         786 :   SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
    2026         786 :   SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
    2027             : 
    2028        2358 :   Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
    2029         786 :   HiOps[2] = Lo.getValue(1);
    2030        2358 :   Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
    2031             : 
    2032             :   // Legalized the flag result - switch anything that used the old flag to
    2033             :   // use the new one.
    2034        1572 :   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
    2035         786 : }
    2036             : 
    2037        3661 : void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
    2038             :                                                SDValue &Lo, SDValue &Hi) {
    2039       10983 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2040             :   SDLoc dl(N);
    2041        3661 :   SDValue Op = N->getOperand(0);
    2042        3661 :   if (Op.getValueType().bitsLE(NVT)) {
    2043             :     // The low part is any extension of the input (which degenerates to a copy).
    2044        7280 :     Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
    2045        3640 :     Hi = DAG.getUNDEF(NVT);   // The high part is undefined.
    2046             :   } else {
    2047             :     // For example, extension of an i48 to an i64.  The operand type necessarily
    2048             :     // promotes to the result type, so will end up being expanded too.
    2049             :     assert(getTypeAction(Op.getValueType()) ==
    2050             :            TargetLowering::TypePromoteInteger &&
    2051             :            "Only know how to promote this result!");
    2052          21 :     SDValue Res = GetPromotedInteger(Op);
    2053             :     assert(Res.getValueType() == N->getValueType(0) &&
    2054             :            "Operand over promoted?");
    2055             :     // Split the promoted operand.  This will simplify when it is expanded.
    2056          21 :     SplitInteger(Res, Lo, Hi);
    2057             :   }
    2058        3661 : }
    2059             : 
    2060         317 : void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
    2061             :                                                SDValue &Lo, SDValue &Hi) {
    2062             :   SDLoc dl(N);
    2063         634 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2064         634 :   EVT NVT = Lo.getValueType();
    2065         634 :   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    2066         317 :   unsigned NVTBits = NVT.getSizeInBits();
    2067         317 :   unsigned EVTBits = EVT.getSizeInBits();
    2068             : 
    2069         317 :   if (NVTBits < EVTBits) {
    2070         508 :     Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
    2071         254 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2072         762 :                                                         EVTBits - NVTBits)));
    2073             :   } else {
    2074         126 :     Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
    2075             :     // The high part replicates the sign bit of Lo, make it explicit.
    2076         126 :     Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
    2077          63 :                      DAG.getConstant(NVTBits - 1, dl,
    2078         189 :                                      TLI.getPointerTy(DAG.getDataLayout())));
    2079             :   }
    2080         317 : }
    2081             : 
    2082         315 : void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
    2083             :                                                SDValue &Lo, SDValue &Hi) {
    2084             :   SDLoc dl(N);
    2085         630 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2086         630 :   EVT NVT = Lo.getValueType();
    2087         630 :   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    2088         315 :   unsigned NVTBits = NVT.getSizeInBits();
    2089         315 :   unsigned EVTBits = EVT.getSizeInBits();
    2090             : 
    2091         315 :   if (NVTBits < EVTBits) {
    2092         504 :     Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
    2093         252 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2094         756 :                                                         EVTBits - NVTBits)));
    2095             :   } else {
    2096         126 :     Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
    2097             :     // The high part must be zero, make it explicit.
    2098          63 :     Hi = DAG.getConstant(0, dl, NVT);
    2099             :   }
    2100         315 : }
    2101             : 
    2102           1 : void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
    2103             :                                                SDValue &Lo, SDValue &Hi) {
    2104             :   SDLoc dl(N);
    2105           2 :   GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
    2106           3 :   Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
    2107           3 :   Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
    2108           1 : }
    2109             : 
    2110          44 : void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
    2111             :                                           SDValue &Lo, SDValue &Hi) {
    2112             :   SDLoc dl(N);
    2113          88 :   GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
    2114         132 :   Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
    2115         132 :   Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
    2116          44 : }
    2117             : 
    2118       68919 : void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
    2119             :                                              SDValue &Lo, SDValue &Hi) {
    2120      206757 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2121       68919 :   unsigned NBitWidth = NVT.getSizeInBits();
    2122             :   auto Constant = cast<ConstantSDNode>(N);
    2123       68919 :   const APInt &Cst = Constant->getAPIntValue();
    2124       68919 :   bool IsTarget = Constant->isTargetOpcode();
    2125             :   bool IsOpaque = Constant->isOpaque();
    2126             :   SDLoc dl(N);
    2127      137838 :   Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
    2128      206757 :   Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
    2129             :                        IsOpaque);
    2130       68919 : }
    2131             : 
    2132          60 : void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
    2133             :                                          SDValue &Lo, SDValue &Hi) {
    2134             :   SDLoc dl(N);
    2135             :   // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
    2136         120 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2137         120 :   EVT NVT = Lo.getValueType();
    2138             : 
    2139          60 :   SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
    2140          60 :                                    DAG.getConstant(0, dl, NVT), ISD::SETNE);
    2141             : 
    2142         180 :   SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
    2143         120 :   SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
    2144             : 
    2145         120 :   Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
    2146             :                      DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
    2147          60 :                                  DAG.getConstant(NVT.getSizeInBits(), dl,
    2148         120 :                                                  NVT)));
    2149          60 :   Hi = DAG.getConstant(0, dl, NVT);
    2150          60 : }
    2151             : 
    2152          12 : void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
    2153             :                                           SDValue &Lo, SDValue &Hi) {
    2154             :   SDLoc dl(N);
    2155             :   // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
    2156          24 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2157          24 :   EVT NVT = Lo.getValueType();
    2158          36 :   Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
    2159          36 :                    DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
    2160          12 :   Hi = DAG.getConstant(0, dl, NVT);
    2161          12 : }
    2162             : 
    2163          14 : void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
    2164             :                                          SDValue &Lo, SDValue &Hi) {
    2165             :   SDLoc dl(N);
    2166             :   // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
    2167          28 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2168          28 :   EVT NVT = Lo.getValueType();
    2169             : 
    2170          14 :   SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
    2171          14 :                                    DAG.getConstant(0, dl, NVT), ISD::SETNE);
    2172             : 
    2173          28 :   SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
    2174          42 :   SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
    2175             : 
    2176          28 :   Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
    2177             :                      DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
    2178          14 :                                  DAG.getConstant(NVT.getSizeInBits(), dl,
    2179          28 :                                                  NVT)));
    2180          14 :   Hi = DAG.getConstant(0, dl, NVT);
    2181          14 : }
    2182             : 
    2183           1 : void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
    2184             :                                                SDValue &Hi) {
    2185             :   SDLoc dl(N);
    2186           3 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2187           1 :   unsigned NBitWidth = NVT.getSizeInBits();
    2188             : 
    2189           2 :   EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
    2190           1 :   Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
    2191             :   // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
    2192           2 :   Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
    2193           2 :                    DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
    2194           1 : }
    2195             : 
    2196          93 : void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
    2197             :                                                SDValue &Hi) {
    2198             :   SDLoc dl(N);
    2199         186 :   EVT VT = N->getValueType(0);
    2200             : 
    2201          93 :   SDValue Op = N->getOperand(0);
    2202         279 :   if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
    2203           6 :     Op = GetPromotedFloat(Op);
    2204             : 
    2205         186 :   RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
    2206             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
    2207         186 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
    2208             :                Lo, Hi);
    2209          93 : }
    2210             : 
    2211          86 : void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
    2212             :                                                SDValue &Hi) {
    2213             :   SDLoc dl(N);
    2214         172 :   EVT VT = N->getValueType(0);
    2215             : 
    2216          86 :   SDValue Op = N->getOperand(0);
    2217         258 :   if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
    2218           4 :     Op = GetPromotedFloat(Op);
    2219             : 
    2220         172 :   RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
    2221             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
    2222         172 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
    2223             :                Lo, Hi);
    2224          86 : }
    2225             : 
    2226       71164 : void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
    2227             :                                          SDValue &Lo, SDValue &Hi) {
    2228             :   if (ISD::isNormalLoad(N)) {
    2229       70411 :     ExpandRes_NormalLoad(N, Lo, Hi);
    2230       70411 :     return;
    2231             :   }
    2232             : 
    2233             :   assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
    2234             : 
    2235         753 :   EVT VT = N->getValueType(0);
    2236        1506 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2237         753 :   SDValue Ch  = N->getChain();
    2238         753 :   SDValue Ptr = N->getBasePtr();
    2239             :   ISD::LoadExtType ExtType = N->getExtensionType();
    2240         753 :   unsigned Alignment = N->getAlignment();
    2241         753 :   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
    2242             :   AAMDNodes AAInfo = N->getAAInfo();
    2243             :   SDLoc dl(N);
    2244             : 
    2245             :   assert(NVT.isByteSized() && "Expanded type not byte sized!");
    2246             : 
    2247         753 :   if (N->getMemoryVT().bitsLE(NVT)) {
    2248         454 :     EVT MemVT = N->getMemoryVT();
    2249             : 
    2250        1362 :     Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
    2251         454 :                         Alignment, MMOFlags, AAInfo);
    2252             : 
    2253             :     // Remember the chain.
    2254         454 :     Ch = Lo.getValue(1);
    2255             : 
    2256         454 :     if (ExtType == ISD::SEXTLOAD) {
    2257             :       // The high part is obtained by SRA'ing all but one of the bits of the
    2258             :       // lo part.
    2259          83 :       unsigned LoSize = Lo.getValueSizeInBits();
    2260         166 :       Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
    2261          83 :                        DAG.getConstant(LoSize - 1, dl,
    2262         249 :                                        TLI.getPointerTy(DAG.getDataLayout())));
    2263         371 :     } else if (ExtType == ISD::ZEXTLOAD) {
    2264             :       // The high part is just a zero.
    2265         244 :       Hi = DAG.getConstant(0, dl, NVT);
    2266             :     } else {
    2267             :       assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
    2268             :       // The high part is undefined.
    2269         127 :       Hi = DAG.getUNDEF(NVT);
    2270             :     }
    2271         598 :   } else if (DAG.getDataLayout().isLittleEndian()) {
    2272             :     // Little-endian - low bits are at low addresses.
    2273         873 :     Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
    2274         291 :                      AAInfo);
    2275             : 
    2276             :     unsigned ExcessBits =
    2277         291 :       N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
    2278         291 :     EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
    2279             : 
    2280             :     // Increment the pointer to the other half.
    2281         291 :     unsigned IncrementSize = NVT.getSizeInBits()/8;
    2282         582 :     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
    2283         582 :                       DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
    2284         582 :     Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
    2285         291 :                         N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
    2286         582 :                         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    2287             : 
    2288             :     // Build a factor node to remember that this load is independent of the
    2289             :     // other one.
    2290         582 :     Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    2291         582 :                      Hi.getValue(1));
    2292             :   } else {
    2293             :     // Big-endian - high bits are at low addresses.  Favor aligned loads at
    2294             :     // the cost of some bit-fiddling.
    2295           8 :     EVT MemVT = N->getMemoryVT();
    2296             :     unsigned EBytes = MemVT.getStoreSize();
    2297           8 :     unsigned IncrementSize = NVT.getSizeInBits()/8;
    2298           8 :     unsigned ExcessBits = (EBytes - IncrementSize)*8;
    2299             : 
    2300             :     // Load both the high bits and maybe some of the low bits.
    2301          24 :     Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
    2302           8 :                         EVT::getIntegerVT(*DAG.getContext(),
    2303             :                                           MemVT.getSizeInBits() - ExcessBits),
    2304          16 :                         Alignment, MMOFlags, AAInfo);
    2305             : 
    2306             :     // Increment the pointer to the other half.
    2307          16 :     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
    2308          16 :                       DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
    2309             :     // Load the rest of the low bits.
    2310          16 :     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
    2311           8 :                         N->getPointerInfo().getWithOffset(IncrementSize),
    2312           8 :                         EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
    2313          24 :                         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    2314             : 
    2315             :     // Build a factor node to remember that this load is independent of the
    2316             :     // other one.
    2317          16 :     Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    2318          16 :                      Hi.getValue(1));
    2319             : 
    2320           8 :     if (ExcessBits < NVT.getSizeInBits()) {
    2321             :       // Transfer low bits from the bottom of Hi to the top of Lo.
    2322          14 :       Lo = DAG.getNode(
    2323             :           ISD::OR, dl, NVT, Lo,
    2324             :           DAG.getNode(ISD::SHL, dl, NVT, Hi,
    2325             :                       DAG.getConstant(ExcessBits, dl,
    2326          21 :                                       TLI.getPointerTy(DAG.getDataLayout()))));
    2327             :       // Move high bits to the right position in Hi.
    2328          14 :       Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
    2329             :                        Hi,
    2330           7 :                        DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
    2331          21 :                                        TLI.getPointerTy(DAG.getDataLayout())));
    2332             :     }
    2333             :   }
    2334             : 
    2335             :   // Legalize the chain result - switch anything that used the old chain to
    2336             :   // use the new one.
    2337         753 :   ReplaceValueWith(SDValue(N, 1), Ch);
    2338             : }
    2339             : 
    2340       17125 : void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
    2341             :                                             SDValue &Lo, SDValue &Hi) {
    2342             :   SDLoc dl(N);
    2343       17125 :   SDValue LL, LH, RL, RH;
    2344       34250 :   GetExpandedInteger(N->getOperand(0), LL, LH);
    2345       34250 :   GetExpandedInteger(N->getOperand(1), RL, RH);
    2346       68500 :   Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
    2347       68500 :   Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
    2348       17125 : }
    2349             : 
    2350        1762 : void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
    2351             :                                         SDValue &Lo, SDValue &Hi) {
    2352        3524 :   EVT VT = N->getValueType(0);
    2353        3524 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2354             :   SDLoc dl(N);
    2355             : 
    2356        1762 :   SDValue LL, LH, RL, RH;
    2357        3524 :   GetExpandedInteger(N->getOperand(0), LL, LH);
    2358        3524 :   GetExpandedInteger(N->getOperand(1), RL, RH);
    2359             : 
    2360        1762 :   if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
    2361             :                     TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
    2362             :                     LL, LH, RL, RH))
    2363             :     return;
    2364             : 
    2365             :   // If nothing else, we can make a libcall.
    2366             :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2367             :   if (VT == MVT::i16)
    2368             :     LC = RTLIB::MUL_I16;
    2369             :   else if (VT == MVT::i32)
    2370             :     LC = RTLIB::MUL_I32;
    2371             :   else if (VT == MVT::i64)
    2372             :     LC = RTLIB::MUL_I64;
    2373             :   else if (VT == MVT::i128)
    2374             :     LC = RTLIB::MUL_I128;
    2375             : 
    2376         512 :   if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
    2377             :     // We'll expand the multiplication by brute force because we have no other
    2378             :     // options. This is a trivially-generalized version of the code from
    2379             :     // Hacker's Delight (itself derived from Knuth's Algorithm M from section
    2380             :     // 4.3.1).
    2381         286 :     unsigned Bits = NVT.getSizeInBits();
    2382         286 :     unsigned HalfBits = Bits >> 1;
    2383         572 :     SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
    2384         286 :                                    NVT);
    2385         572 :     SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
    2386         572 :     SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
    2387             : 
    2388         572 :     SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
    2389         572 :     SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
    2390             : 
    2391         572 :     EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
    2392         858 :     if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
    2393             :       // The type from TLI is too small to fit the shift amount we want.
    2394             :       // Override it with i32. The shift will have to be legalized.
    2395           2 :       ShiftAmtTy = MVT::i32;
    2396             :     }
    2397         286 :     SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
    2398         572 :     SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
    2399         572 :     SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
    2400         572 :     SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
    2401             : 
    2402         286 :     SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
    2403         286 :                             DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
    2404         572 :     SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
    2405         572 :     SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
    2406             : 
    2407         286 :     SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
    2408         286 :                             DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
    2409         572 :     SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
    2410             : 
    2411         286 :     SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
    2412         286 :                             DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
    2413         572 :                             DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
    2414         572 :     Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
    2415         572 :                      DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
    2416             : 
    2417         572 :     Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
    2418             :                      DAG.getNode(ISD::ADD, dl, NVT,
    2419         286 :                                  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
    2420         858 :                                  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
    2421             :     return;
    2422             :   }
    2423             : 
    2424          64 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2425          64 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
    2426             :                Lo, Hi);
    2427             : }
    2428             : 
    2429           2 : void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
    2430             :                                                      SDValue &Hi) {
    2431             :   SDLoc DL(N);
    2432           6 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2433           4 :   SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
    2434           6 :   SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
    2435           2 :   Lo = R.getValue(0);
    2436           2 :   Hi = R.getValue(1);
    2437           2 :   ReplaceValueWith(SDValue(N, 1), R.getValue(2));
    2438           2 : }
    2439             : 
    2440          20 : void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
    2441             :                                              SDValue &Lo, SDValue &Hi) {
    2442          20 :   SDValue LHS = Node->getOperand(0);
    2443          20 :   SDValue RHS = Node->getOperand(1);
    2444             :   SDLoc dl(Node);
    2445             : 
    2446             :   // Expand the result by simply replacing it with the equivalent
    2447             :   // non-overflow-checking operation.
    2448          40 :   SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
    2449             :                             ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
    2450          20 :                             LHS, RHS);
    2451          20 :   SplitInteger(Sum, Lo, Hi);
    2452             : 
    2453             :   // Compute the overflow.
    2454             :   //
    2455             :   //   LHSSign -> LHS >= 0
    2456             :   //   RHSSign -> RHS >= 0
    2457             :   //   SumSign -> Sum >= 0
    2458             :   //
    2459             :   //   Add:
    2460             :   //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
    2461             :   //   Sub:
    2462             :   //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
    2463             :   //
    2464          40 :   EVT OType = Node->getValueType(1);
    2465          40 :   SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
    2466             : 
    2467          20 :   SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
    2468          20 :   SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
    2469          20 :   SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
    2470          20 :                                     Node->getOpcode() == ISD::SADDO ?
    2471          20 :                                     ISD::SETEQ : ISD::SETNE);
    2472             : 
    2473          20 :   SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
    2474          20 :   SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
    2475             : 
    2476          40 :   SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
    2477             : 
    2478             :   // Use the calculated overflow everywhere.
    2479          20 :   ReplaceValueWith(SDValue(Node, 1), Cmp);
    2480          20 : }
    2481             : 
    2482          69 : void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
    2483             :                                          SDValue &Lo, SDValue &Hi) {
    2484         138 :   EVT VT = N->getValueType(0);
    2485             :   SDLoc dl(N);
    2486          69 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2487             : 
    2488         138 :   if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
    2489          12 :     SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2490           6 :     SplitInteger(Res.getValue(0), Lo, Hi);
    2491             :     return;
    2492             :   }
    2493             : 
    2494             :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2495             :   if (VT == MVT::i16)
    2496             :     LC = RTLIB::SDIV_I16;
    2497             :   else if (VT == MVT::i32)
    2498             :     LC = RTLIB::SDIV_I32;
    2499             :   else if (VT == MVT::i64)
    2500             :     LC = RTLIB::SDIV_I64;
    2501             :   else if (VT == MVT::i128)
    2502             :     LC = RTLIB::SDIV_I128;
    2503             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
    2504             : 
    2505         126 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
    2506             : }
    2507             : 
    2508       65941 : void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
    2509             :                                           SDValue &Lo, SDValue &Hi) {
    2510      131882 :   EVT VT = N->getValueType(0);
    2511             :   SDLoc dl(N);
    2512             : 
    2513             :   // If we can emit an efficient shift operation, do so now.  Check to see if
    2514             :   // the RHS is a constant.
    2515       65941 :   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    2516      130382 :     return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
    2517             : 
    2518             :   // If we can determine that the high bit of the shift is zero or one, even if
    2519             :   // the low bits are variable, emit this shift in an optimized form.
    2520         750 :   if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
    2521             :     return;
    2522             : 
    2523             :   // If this target supports shift_PARTS, use it.  First, map to the _PARTS opc.
    2524             :   unsigned PartsOpc;
    2525        1474 :   if (N->getOpcode() == ISD::SHL) {
    2526             :     PartsOpc = ISD::SHL_PARTS;
    2527         373 :   } else if (N->getOpcode() == ISD::SRL) {
    2528             :     PartsOpc = ISD::SRL_PARTS;
    2529             :   } else {
    2530             :     assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
    2531             :     PartsOpc = ISD::SRA_PARTS;
    2532             :   }
    2533             : 
    2534             :   // Next check to see if the target supports this SHL_PARTS operation or if it
    2535             :   // will custom expand it.
    2536         737 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2537         737 :   TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
    2538        1469 :   if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
    2539             :       Action == TargetLowering::Custom) {
    2540             :     // Expand the subcomponents.
    2541         569 :     SDValue LHSL, LHSH;
    2542        1138 :     GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    2543        1138 :     EVT VT = LHSL.getValueType();
    2544             : 
    2545             :     // If the shift amount operand is coming from a vector legalization it may
    2546             :     // have an illegal type.  Fix that first by casting the operand, otherwise
    2547             :     // the new SHL_PARTS operation would need further legalization.
    2548         569 :     SDValue ShiftOp = N->getOperand(1);
    2549        1138 :     EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2550             :     assert(ShiftTy.getScalarSizeInBits() >=
    2551             :            Log2_32_Ceil(VT.getScalarSizeInBits()) &&
    2552             :            "ShiftAmountTy is too small to cover the range of this type!");
    2553         569 :     if (ShiftOp.getValueType() != ShiftTy)
    2554          55 :       ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
    2555             : 
    2556         569 :     SDValue Ops[] = { LHSL, LHSH, ShiftOp };
    2557        1138 :     Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
    2558         569 :     Hi = Lo.getValue(1);
    2559             :     return;
    2560             :   }
    2561             : 
    2562             :   // Otherwise, emit a libcall.
    2563             :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2564             :   bool isSigned;
    2565         336 :   if (N->getOpcode() == ISD::SHL) {
    2566             :     isSigned = false; /*sign irrelevant*/
    2567             :     if (VT == MVT::i16)
    2568             :       LC = RTLIB::SHL_I16;
    2569             :     else if (VT == MVT::i32)
    2570             :       LC = RTLIB::SHL_I32;
    2571             :     else if (VT == MVT::i64)
    2572             :       LC = RTLIB::SHL_I64;
    2573             :     else if (VT == MVT::i128)
    2574             :       LC = RTLIB::SHL_I128;
    2575          83 :   } else if (N->getOpcode() == ISD::SRL) {
    2576             :     isSigned = false;
    2577             :     if (VT == MVT::i16)
    2578             :       LC = RTLIB::SRL_I16;
    2579             :     else if (VT == MVT::i32)
    2580             :       LC = RTLIB::SRL_I32;
    2581             :     else if (VT == MVT::i64)
    2582             :       LC = RTLIB::SRL_I64;
    2583             :     else if (VT == MVT::i128)
    2584             :       LC = RTLIB::SRL_I128;
    2585             :   } else {
    2586             :     assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
    2587             :     isSigned = true;
    2588             :     if (VT == MVT::i16)
    2589             :       LC = RTLIB::SRA_I16;
    2590             :     else if (VT == MVT::i32)
    2591             :       LC = RTLIB::SRA_I32;
    2592             :     else if (VT == MVT::i64)
    2593             :       LC = RTLIB::SRA_I64;
    2594             :     else if (VT == MVT::i128)
    2595             :       LC = RTLIB::SRA_I128;
    2596             :   }
    2597             : 
    2598         258 :   if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
    2599          83 :     SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2600         166 :     SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
    2601             :     return;
    2602             :   }
    2603             : 
    2604          85 :   if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
    2605           0 :     llvm_unreachable("Unsupported shift!");
    2606             : }
    2607             : 
    2608        1678 : void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
    2609             :                                                 SDValue &Lo, SDValue &Hi) {
    2610        5034 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2611             :   SDLoc dl(N);
    2612        1678 :   SDValue Op = N->getOperand(0);
    2613        3356 :   if (Op.getValueType().bitsLE(NVT)) {
    2614             :     // The low part is sign extension of the input (degenerates to a copy).
    2615        3354 :     Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
    2616             :     // The high part is obtained by SRA'ing all but one of the bits of low part.
    2617        1677 :     unsigned LoSize = NVT.getSizeInBits();
    2618        3354 :     Hi = DAG.getNode(
    2619             :         ISD::SRA, dl, NVT, Lo,
    2620        5031 :         DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
    2621             :   } else {
    2622             :     // For example, extension of an i48 to an i64.  The operand type necessarily
    2623             :     // promotes to the result type, so will end up being expanded too.
    2624             :     assert(getTypeAction(Op.getValueType()) ==
    2625             :            TargetLowering::TypePromoteInteger &&
    2626             :            "Only know how to promote this result!");
    2627           1 :     SDValue Res = GetPromotedInteger(Op);
    2628             :     assert(Res.getValueType() == N->getValueType(0) &&
    2629             :            "Operand over promoted?");
    2630             :     // Split the promoted operand.  This will simplify when it is expanded.
    2631           1 :     SplitInteger(Res, Lo, Hi);
    2632           1 :     unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
    2633           2 :     Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
    2634           1 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2635           2 :                                                         ExcessBits)));
    2636             :   }
    2637        1678 : }
    2638             : 
    2639         159 : void DAGTypeLegalizer::
    2640             : ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
    2641             :   SDLoc dl(N);
    2642         318 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2643         318 :   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    2644             : 
    2645         318 :   if (EVT.bitsLE(Lo.getValueType())) {
    2646             :     // sext_inreg the low part if needed.
    2647         180 :     Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
    2648          90 :                      N->getOperand(1));
    2649             : 
    2650             :     // The high part gets the sign extension from the lo-part.  This handles
    2651             :     // things like sextinreg V:i64 from i8.
    2652         180 :     Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
    2653          90 :                      DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
    2654         270 :                                      TLI.getPointerTy(DAG.getDataLayout())));
    2655             :   } else {
    2656             :     // For example, extension of an i48 to an i64.  Leave the low part alone,
    2657             :     // sext_inreg the high part.
    2658          69 :     unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
    2659         138 :     Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
    2660          69 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2661         138 :                                                         ExcessBits)));
    2662             :   }
    2663         159 : }
    2664             : 
    2665          74 : void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
    2666             :                                          SDValue &Lo, SDValue &Hi) {
    2667         148 :   EVT VT = N->getValueType(0);
    2668             :   SDLoc dl(N);
    2669          74 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2670             : 
    2671         148 :   if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
    2672          20 :     SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2673          10 :     SplitInteger(Res.getValue(1), Lo, Hi);
    2674             :     return;
    2675             :   }
    2676             : 
    2677             :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2678             :   if (VT == MVT::i16)
    2679             :     LC = RTLIB::SREM_I16;
    2680             :   else if (VT == MVT::i32)
    2681             :     LC = RTLIB::SREM_I32;
    2682             :   else if (VT == MVT::i64)
    2683             :     LC = RTLIB::SREM_I64;
    2684             :   else if (VT == MVT::i128)
    2685             :     LC = RTLIB::SREM_I128;
    2686             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
    2687             : 
    2688         128 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
    2689             : }
    2690             : 
    2691       13613 : void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
    2692             :                                              SDValue &Lo, SDValue &Hi) {
    2693       40839 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2694             :   SDLoc dl(N);
    2695       40839 :   Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
    2696       27226 :   Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
    2697       13613 :                    N->getOperand(0),
    2698       13613 :                    DAG.getConstant(NVT.getSizeInBits(), dl,
    2699       54452 :                                    TLI.getPointerTy(DAG.getDataLayout())));
    2700       27226 :   Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
    2701       13613 : }
    2702             : 
    2703          12 : void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
    2704             :                                           SDValue &Lo, SDValue &Hi) {
    2705          24 :   EVT VT = N->getValueType(0);
    2706             :   SDLoc dl(N);
    2707             : 
    2708             :   // A divide for UMULO should be faster than a function call.
    2709          12 :   if (N->getOpcode() == ISD::UMULO) {
    2710           2 :     SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
    2711             : 
    2712           4 :     SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
    2713           2 :     SplitInteger(MUL, Lo, Hi);
    2714             : 
    2715             :     // A divide for UMULO will be faster than a function call. Select to
    2716             :     // make sure we aren't using 0.
    2717           2 :     SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
    2718           2 :                                   RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
    2719           2 :     SDValue NotZero = DAG.getSelect(dl, VT, isZero,
    2720           2 :                                     DAG.getConstant(1, dl, VT), RHS);
    2721           4 :     SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
    2722           2 :     SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
    2723           4 :                                     ISD::SETNE);
    2724           4 :     Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
    2725             :                              DAG.getConstant(0, dl, N->getValueType(1)),
    2726           4 :                              Overflow);
    2727           2 :     ReplaceValueWith(SDValue(N, 1), Overflow);
    2728             :     return;
    2729             :   }
    2730             : 
    2731          10 :   Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
    2732          10 :   EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
    2733          10 :   Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
    2734             : 
    2735             :   // Replace this with a libcall that will check overflow.
    2736             :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2737             :   if (VT == MVT::i32)
    2738             :     LC = RTLIB::MULO_I32;
    2739             :   else if (VT == MVT::i64)
    2740             :     LC = RTLIB::MULO_I64;
    2741             :   else if (VT == MVT::i128)
    2742             :     LC = RTLIB::MULO_I128;
    2743             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
    2744             : 
    2745          10 :   SDValue Temp = DAG.CreateStackTemporary(PtrVT);
    2746             :   // Temporary for the overflow value, default it to zero.
    2747             :   SDValue Chain =
    2748          20 :       DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
    2749          20 :                    MachinePointerInfo());
    2750             : 
    2751             :   TargetLowering::ArgListTy Args;
    2752             :   TargetLowering::ArgListEntry Entry;
    2753          30 :   for (const SDValue &Op : N->op_values()) {
    2754          40 :     EVT ArgVT = Op.getValueType();
    2755          20 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2756          20 :     Entry.Node = Op;
    2757          20 :     Entry.Ty = ArgTy;
    2758          20 :     Entry.IsSExt = true;
    2759          20 :     Entry.IsZExt = false;
    2760          20 :     Args.push_back(Entry);
    2761             :   }
    2762             : 
    2763             :   // Also pass the address of the overflow check.
    2764          10 :   Entry.Node = Temp;
    2765          10 :   Entry.Ty = PtrTy->getPointerTo();
    2766          10 :   Entry.IsSExt = true;
    2767          10 :   Entry.IsZExt = false;
    2768          10 :   Args.push_back(Entry);
    2769             : 
    2770          20 :   SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
    2771             : 
    2772          20 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2773             :   CLI.setDebugLoc(dl)
    2774             :       .setChain(Chain)
    2775          20 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
    2776             :       .setSExtResult();
    2777             : 
    2778          10 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2779             : 
    2780          10 :   SplitInteger(CallInfo.first, Lo, Hi);
    2781             :   SDValue Temp2 =
    2782          20 :       DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
    2783          10 :   SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
    2784             :                              DAG.getConstant(0, dl, PtrVT),
    2785          20 :                              ISD::SETNE);
    2786             :   // Use the overflow from the libcall everywhere.
    2787          10 :   ReplaceValueWith(SDValue(N, 1), Ofl);
    2788             : }
    2789             : 
    2790          68 : void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
    2791             :                                          SDValue &Lo, SDValue &Hi) {
    2792         136 :   EVT VT = N->getValueType(0);
    2793             :   SDLoc dl(N);
    2794          68 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2795             : 
    2796         136 :   if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
    2797          26 :     SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2798          13 :     SplitInteger(Res.getValue(0), Lo, Hi);
    2799             :     return;
    2800             :   }
    2801             : 
    2802             :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2803             :   if (VT == MVT::i16)
    2804             :     LC = RTLIB::UDIV_I16;
    2805             :   else if (VT == MVT::i32)
    2806             :     LC = RTLIB::UDIV_I32;
    2807             :   else if (VT == MVT::i64)
    2808             :     LC = RTLIB::UDIV_I64;
    2809             :   else if (VT == MVT::i128)
    2810             :     LC = RTLIB::UDIV_I128;
    2811             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
    2812             : 
    2813         110 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
    2814             : }
    2815             : 
    2816          75 : void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
    2817             :                                          SDValue &Lo, SDValue &Hi) {
    2818         150 :   EVT VT = N->getValueType(0);
    2819             :   SDLoc dl(N);
    2820          75 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2821             : 
    2822         150 :   if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
    2823          24 :     SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2824          12 :     SplitInteger(Res.getValue(1), Lo, Hi);
    2825             :     return;
    2826             :   }
    2827             : 
    2828             :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2829             :   if (VT == MVT::i16)
    2830             :     LC = RTLIB::UREM_I16;
    2831             :   else if (VT == MVT::i32)
    2832             :     LC = RTLIB::UREM_I32;
    2833             :   else if (VT == MVT::i64)
    2834             :     LC = RTLIB::UREM_I64;
    2835             :   else if (VT == MVT::i128)
    2836             :     LC = RTLIB::UREM_I128;
    2837             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
    2838             : 
    2839         126 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
    2840             : }
    2841             : 
    2842        5840 : void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
    2843             :                                                 SDValue &Lo, SDValue &Hi) {
    2844       17520 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2845             :   SDLoc dl(N);
    2846        5840 :   SDValue Op = N->getOperand(0);
    2847       11680 :   if (Op.getValueType().bitsLE(NVT)) {
    2848             :     // The low part is zero extension of the input (degenerates to a copy).
    2849       11680 :     Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
    2850        5840 :     Hi = DAG.getConstant(0, dl, NVT);   // The high part is just a zero.
    2851             :   } else {
    2852             :     // For example, extension of an i48 to an i64.  The operand type necessarily
    2853             :     // promotes to the result type, so will end up being expanded too.
    2854             :     assert(getTypeAction(Op.getValueType()) ==
    2855             :            TargetLowering::TypePromoteInteger &&
    2856             :            "Only know how to promote this result!");
    2857           0 :     SDValue Res = GetPromotedInteger(Op);
    2858             :     assert(Res.getValueType() == N->getValueType(0) &&
    2859             :            "Operand over promoted?");
    2860             :     // Split the promoted operand.  This will simplify when it is expanded.
    2861           0 :     SplitInteger(Res, Lo, Hi);
    2862           0 :     unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
    2863           0 :     Hi = DAG.getZeroExtendInReg(Hi, dl,
    2864           0 :                                 EVT::getIntegerVT(*DAG.getContext(),
    2865           0 :                                                   ExcessBits));
    2866             :   }
    2867        5840 : }
    2868             : 
    2869          13 : void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
    2870             :                                                 SDValue &Lo, SDValue &Hi) {
    2871             :   SDLoc dl(N);
    2872          13 :   EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
    2873          26 :   SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
    2874          13 :   SDValue Zero = DAG.getConstant(0, dl, VT);
    2875          13 :   SDValue Swap = DAG.getAtomicCmpSwap(
    2876             :       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
    2877             :       cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
    2878          26 :       N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
    2879             : 
    2880          13 :   ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
    2881          13 :   ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
    2882          13 : }
    2883             : 
    2884             : //===----------------------------------------------------------------------===//
    2885             : //  Integer Operand Expansion
    2886             : //===----------------------------------------------------------------------===//
    2887             : 
    2888             : /// ExpandIntegerOperand - This method is called when the specified operand of
    2889             : /// the specified node is found to need expansion.  At this point, all of the
    2890             : /// result types of the node are known to be legal, but other operands of the
    2891             : /// node may need promotion or expansion as well as the specified one.
    2892      214312 : bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
    2893             :   LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
    2894             :              dbgs() << "\n");
    2895      214312 :   SDValue Res = SDValue();
    2896             : 
    2897      642936 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    2898             :     return false;
    2899             : 
    2900      427522 :   switch (N->getOpcode()) {
    2901           0 :   default:
    2902             :   #ifndef NDEBUG
    2903             :     dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
    2904             :     N->dump(&DAG); dbgs() << "\n";
    2905             :   #endif
    2906           0 :     llvm_unreachable("Do not know how to expand this operator's operand!");
    2907             : 
    2908        3951 :   case ISD::BITCAST:           Res = ExpandOp_BITCAST(N); break;
    2909           0 :   case ISD::BR_CC:             Res = ExpandIntOp_BR_CC(N); break;
    2910       17554 :   case ISD::BUILD_VECTOR:      Res = ExpandOp_BUILD_VECTOR(N); break;
    2911       17918 :   case ISD::EXTRACT_ELEMENT:   Res = ExpandOp_EXTRACT_ELEMENT(N); break;
    2912          30 :   case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
    2913          62 :   case ISD::SCALAR_TO_VECTOR:  Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
    2914        1754 :   case ISD::SELECT_CC:         Res = ExpandIntOp_SELECT_CC(N); break;
    2915         849 :   case ISD::SETCC:             Res = ExpandIntOp_SETCC(N); break;
    2916          24 :   case ISD::SETCCCARRY:        Res = ExpandIntOp_SETCCCARRY(N); break;
    2917          42 :   case ISD::SINT_TO_FP:        Res = ExpandIntOp_SINT_TO_FP(N); break;
    2918       71777 :   case ISD::STORE:   Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
    2919       97276 :   case ISD::TRUNCATE:          Res = ExpandIntOp_TRUNCATE(N); break;
    2920          40 :   case ISD::UINT_TO_FP:        Res = ExpandIntOp_UINT_TO_FP(N); break;
    2921             : 
    2922        2470 :   case ISD::SHL:
    2923             :   case ISD::SRA:
    2924             :   case ISD::SRL:
    2925             :   case ISD::ROTL:
    2926        2470 :   case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
    2927           1 :   case ISD::RETURNADDR:
    2928           1 :   case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;
    2929             : 
    2930          13 :   case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
    2931             :   }
    2932             : 
    2933             :   // If the result is null, the sub-method took care of registering results etc.
    2934      213761 :   if (!Res.getNode()) return false;
    2935             : 
    2936             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    2937             :   // core about this.
    2938      213761 :   if (Res.getNode() == N)
    2939             :     return true;
    2940             : 
    2941             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    2942             :          "Invalid operand expansion");
    2943             : 
    2944      209055 :   ReplaceValueWith(SDValue(N, 0), Res);
    2945      209055 :   return false;
    2946             : }
    2947             : 
    2948             : /// IntegerExpandSetCCOperands - Expand the operands of a comparison.  This code
    2949             : /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
    2950        2603 : void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
    2951             :                                                   SDValue &NewRHS,
    2952             :                                                   ISD::CondCode &CCCode,
    2953             :                                                   const SDLoc &dl) {
    2954        2603 :   SDValue LHSLo, LHSHi, RHSLo, RHSHi;
    2955        2603 :   GetExpandedInteger(NewLHS, LHSLo, LHSHi);
    2956        2603 :   GetExpandedInteger(NewRHS, RHSLo, RHSHi);
    2957             : 
    2958        2603 :   if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
    2959             :     if (RHSLo == RHSHi) {
    2960             :       if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
    2961         414 :         if (RHSCST->isAllOnesValue()) {
    2962             :           // Equality comparison to -1.
    2963           8 :           NewLHS = DAG.getNode(ISD::AND, dl,
    2964           8 :                                LHSLo.getValueType(), LHSLo, LHSHi);
    2965           4 :           NewRHS = RHSLo;
    2966         761 :           return;
    2967             :         }
    2968             :       }
    2969             :     }
    2970             : 
    2971        1059 :     NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
    2972        1059 :     NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
    2973        1059 :     NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
    2974         706 :     NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
    2975         353 :     return;
    2976             :   }
    2977             : 
    2978             :   // If this is a comparison of the sign bit, just look at the top part.
    2979             :   // X > -1,  x < 0
    2980             :   if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
    2981         461 :     if ((CCCode == ISD::SETLT && CST->isNullValue()) ||     // X < 0
    2982          96 :         (CCCode == ISD::SETGT && CST->isAllOnesValue())) {  // X > -1
    2983         144 :       NewLHS = LHSHi;
    2984         144 :       NewRHS = RHSHi;
    2985         144 :       return;
    2986             :     }
    2987             : 
    2988             :   // FIXME: This generated code sucks.
    2989             :   ISD::CondCode LowCC;
    2990        2102 :   switch (CCCode) {
    2991           0 :   default: llvm_unreachable("Unknown integer setcc!");
    2992             :   case ISD::SETLT:
    2993             :   case ISD::SETULT: LowCC = ISD::SETULT; break;
    2994         112 :   case ISD::SETGT:
    2995         112 :   case ISD::SETUGT: LowCC = ISD::SETUGT; break;
    2996          31 :   case ISD::SETLE:
    2997          31 :   case ISD::SETULE: LowCC = ISD::SETULE; break;
    2998        1808 :   case ISD::SETGE:
    2999        1808 :   case ISD::SETUGE: LowCC = ISD::SETUGE; break;
    3000             :   }
    3001             : 
    3002             :   // LoCmp = lo(op1) < lo(op2)   // Always unsigned comparison
    3003             :   // HiCmp = hi(op1) < hi(op2)   // Signedness depends on operands
    3004             :   // dest  = hi(op1) == hi(op2) ? LoCmp : HiCmp;
    3005             : 
    3006             :   // NOTE: on targets without efficient SELECT of bools, we can always use
    3007             :   // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
    3008             :   TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
    3009        2102 :                                                  nullptr);
    3010             :   SDValue LoCmp, HiCmp;
    3011        2102 :   if (TLI.isTypeLegal(LHSLo.getValueType()) &&
    3012        2080 :       TLI.isTypeLegal(RHSLo.getValueType()))
    3013        2080 :     LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
    3014        2080 :                               RHSLo, LowCC, false, DagCombineInfo, dl);
    3015        2102 :   if (!LoCmp.getNode())
    3016        4056 :     LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
    3017        4056 :                          RHSLo, LowCC);
    3018        2102 :   if (TLI.isTypeLegal(LHSHi.getValueType()) &&
    3019        2080 :       TLI.isTypeLegal(RHSHi.getValueType()))
    3020        2080 :     HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
    3021        2080 :                               RHSHi, CCCode, false, DagCombineInfo, dl);
    3022        2102 :   if (!HiCmp.getNode())
    3023        1975 :     HiCmp =
    3024        5925 :         DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
    3025        5925 :                     LHSHi, RHSHi, DAG.getCondCode(CCCode));
    3026             : 
    3027             :   ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
    3028             :   ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
    3029             : 
    3030        4131 :   bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
    3031        2377 :                     CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
    3032             : 
    3033        2107 :   if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
    3034         303 :       (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
    3035           6 :                       (LoCmpC && LoCmpC->isNullValue())))) {
    3036             :     // For LE / GE, if high part is known false, ignore the low part.
    3037             :     // For LT / GT: if low part is known false, return the high part.
    3038             :     //              if high part is known true, ignore the low part.
    3039           5 :     NewLHS = HiCmp;
    3040           5 :     NewRHS = SDValue();
    3041           5 :     return;
    3042             :   }
    3043             : 
    3044             :   if (LHSHi == RHSHi) {
    3045             :     // Comparing the low bits is enough.
    3046           8 :     NewLHS = LoCmp;
    3047           8 :     NewRHS = SDValue();
    3048           8 :     return;
    3049             :   }
    3050             : 
    3051             :   // Lower with SETCCCARRY if the target supports it.
    3052        4178 :   EVT HiVT = LHSHi.getValueType();
    3053        2089 :   EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
    3054        2089 :   bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
    3055             : 
    3056             :   // FIXME: Make all targets support this, then remove the other lowering.
    3057             :   if (HasSETCCCARRY) {
    3058             :     // SETCCCARRY can detect < and >= directly. For > and <=, flip
    3059             :     // operands and condition code.
    3060             :     bool FlipOperands = false;
    3061         243 :     switch (CCCode) {
    3062          35 :     case ISD::SETGT:  CCCode = ISD::SETLT;  FlipOperands = true; break;
    3063          45 :     case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
    3064          12 :     case ISD::SETLE:  CCCode = ISD::SETGE;  FlipOperands = true; break;
    3065          10 :     case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
    3066             :     default: break;
    3067             :     }
    3068             :     if (FlipOperands) {
    3069             :       std::swap(LHSLo, RHSLo);
    3070             :       std::swap(LHSHi, RHSHi);
    3071             :     }
    3072             :     // Perform a wide subtraction, feeding the carry from the low part into
    3073             :     // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
    3074             :     // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
    3075             :     // zero or positive iff LHS >= RHS.
    3076         486 :     EVT LoVT = LHSLo.getValueType();
    3077         243 :     SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
    3078         243 :     SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
    3079         243 :     SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
    3080             :                               LHSHi, RHSHi, LowCmp.getValue(1),
    3081         243 :                               DAG.getCondCode(CCCode));
    3082         243 :     NewLHS = Res;
    3083         243 :     NewRHS = SDValue();
    3084             :     return;
    3085             :   }
    3086             : 
    3087        1846 :   NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
    3088        1846 :                              false, DagCombineInfo, dl);
    3089        1846 :   if (!NewLHS.getNode())
    3090        1817 :     NewLHS =
    3091        3634 :         DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
    3092        3692 :   NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
    3093        1846 :   NewRHS = SDValue();
    3094             : }
    3095             : 
    3096           0 : SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
    3097           0 :   SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
    3098           0 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
    3099           0 :   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
    3100             : 
    3101             :   // If ExpandSetCCOperands returned a scalar, we need to compare the result
    3102             :   // against zero to select between true and false values.
    3103           0 :   if (!NewRHS.getNode()) {
    3104           0 :     NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
    3105           0 :     CCCode = ISD::SETNE;
    3106             :   }
    3107             : 
    3108             :   // Update N to have the operands specified.
    3109           0 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    3110             :                                 DAG.getCondCode(CCCode), NewLHS, NewRHS,
    3111           0 :                                 N->getOperand(4)), 0);
    3112             : }
    3113             : 
    3114        1754 : SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
    3115        1754 :   SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
    3116        1754 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
    3117        3508 :   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
    3118             : 
    3119             :   // If ExpandSetCCOperands returned a scalar, we need to compare the result
    3120             :   // against zero to select between true and false values.
    3121        1754 :   if (!NewRHS.getNode()) {
    3122        5202 :     NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
    3123        1734 :     CCCode = ISD::SETNE;
    3124             :   }
    3125             : 
    3126             :   // Update N to have the operands specified.
    3127        3508 :   return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
    3128        1754 :                                 N->getOperand(2), N->getOperand(3),
    3129        1754 :                                 DAG.getCondCode(CCCode)), 0);
    3130             : }
    3131             : 
    3132         849 : SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
    3133         849 :   SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
    3134         849 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
    3135        1698 :   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
    3136             : 
    3137             :   // If ExpandSetCCOperands returned a scalar, use it.
    3138         849 :   if (!NewRHS.getNode()) {
    3139             :     assert(NewLHS.getValueType() == N->getValueType(0) &&
    3140             :            "Unexpected setcc expansion!");
    3141         368 :     return NewLHS;
    3142             :   }
    3143             : 
    3144             :   // Otherwise, update N to have the operands specified.
    3145         481 :   return SDValue(
    3146         962 :       DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
    3147             : }
    3148             : 
    3149          24 : SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
    3150          24 :   SDValue LHS = N->getOperand(0);
    3151          24 :   SDValue RHS = N->getOperand(1);
    3152          24 :   SDValue Carry = N->getOperand(2);
    3153          24 :   SDValue Cond = N->getOperand(3);
    3154             :   SDLoc dl = SDLoc(N);
    3155             : 
    3156          24 :   SDValue LHSLo, LHSHi, RHSLo, RHSHi;
    3157          24 :   GetExpandedInteger(LHS, LHSLo, LHSHi);
    3158          24 :   GetExpandedInteger(RHS, RHSLo, RHSHi);
    3159             : 
    3160             :   // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
    3161          72 :   SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
    3162          24 :   SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
    3163          24 :   return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
    3164          72 :                      LowCmp.getValue(1), Cond);
    3165             : }
    3166             : 
    3167        2470 : SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
    3168             :   // The value being shifted is legal, but the shift amount is too big.
    3169             :   // It follows that either the result of the shift is undefined, or the
    3170             :   // upper half of the shift amount is zero.  Just use the lower half.
    3171        2470 :   SDValue Lo, Hi;
    3172        4940 :   GetExpandedInteger(N->getOperand(1), Lo, Hi);
    3173        4940 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
    3174             : }
    3175             : 
    3176           1 : SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
    3177             :   // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant.  This
    3178             :   // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
    3179             :   // constant to valid type.
    3180           1 :   SDValue Lo, Hi;
    3181           2 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    3182           1 :   return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
    3183             : }
    3184             : 
    3185          42 : SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
    3186          42 :   SDValue Op = N->getOperand(0);
    3187          84 :   EVT DstVT = N->getValueType(0);
    3188          84 :   RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
    3189             :   assert(LC != RTLIB::UNKNOWN_LIBCALL &&
    3190             :          "Don't know how to expand this SINT_TO_FP!");
    3191         126 :   return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
    3192             : }
    3193             : 
    3194       71777 : SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
    3195             :   if (ISD::isNormalStore(N))
    3196       70373 :     return ExpandOp_NormalStore(N, OpNo);
    3197             : 
    3198             :   assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
    3199             :   assert(OpNo == 1 && "Can only expand the stored value so far");
    3200             : 
    3201        1404 :   EVT VT = N->getOperand(1).getValueType();
    3202        2808 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3203        1404 :   SDValue Ch  = N->getChain();
    3204        1404 :   SDValue Ptr = N->getBasePtr();
    3205        1404 :   unsigned Alignment = N->getAlignment();
    3206        1404 :   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
    3207             :   AAMDNodes AAInfo = N->getAAInfo();
    3208             :   SDLoc dl(N);
    3209        1404 :   SDValue Lo, Hi;
    3210             : 
    3211             :   assert(NVT.isByteSized() && "Expanded type not byte sized!");
    3212             : 
    3213        1404 :   if (N->getMemoryVT().bitsLE(NVT)) {
    3214         336 :     GetExpandedInteger(N->getValue(), Lo, Hi);
    3215         672 :     return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
    3216         672 :                              N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
    3217             :   }
    3218             : 
    3219        2136 :   if (DAG.getDataLayout().isLittleEndian()) {
    3220             :     // Little-endian - low bits are at low addresses.
    3221        1058 :     GetExpandedInteger(N->getValue(), Lo, Hi);
    3222             : 
    3223        3174 :     Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
    3224        1058 :                       AAInfo);
    3225             : 
    3226             :     unsigned ExcessBits =
    3227        1058 :       N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
    3228        1058 :     EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
    3229             : 
    3230             :     // Increment the pointer to the other half.
    3231        1058 :     unsigned IncrementSize = NVT.getSizeInBits()/8;
    3232        1058 :     Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
    3233        2116 :     Hi = DAG.getTruncStore(
    3234        1058 :         Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
    3235        3174 :         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    3236        2116 :     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
    3237             :   }
    3238             : 
    3239             :   // Big-endian - high bits are at low addresses.  Favor aligned stores at
    3240             :   // the cost of some bit-fiddling.
    3241          10 :   GetExpandedInteger(N->getValue(), Lo, Hi);
    3242             : 
    3243          10 :   EVT ExtVT = N->getMemoryVT();
    3244             :   unsigned EBytes = ExtVT.getStoreSize();
    3245          10 :   unsigned IncrementSize = NVT.getSizeInBits()/8;
    3246          10 :   unsigned ExcessBits = (EBytes - IncrementSize)*8;
    3247          10 :   EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
    3248          20 :                                ExtVT.getSizeInBits() - ExcessBits);
    3249             : 
    3250          10 :   if (ExcessBits < NVT.getSizeInBits()) {
    3251             :     // Transfer high bits from the top of Lo to the bottom of Hi.
    3252          16 :     Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
    3253           8 :                      DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
    3254          24 :                                      TLI.getPointerTy(DAG.getDataLayout())));
    3255          16 :     Hi = DAG.getNode(
    3256             :         ISD::OR, dl, NVT, Hi,
    3257             :         DAG.getNode(ISD::SRL, dl, NVT, Lo,
    3258             :                     DAG.getConstant(ExcessBits, dl,
    3259          24 :                                     TLI.getPointerTy(DAG.getDataLayout()))));
    3260             :   }
    3261             : 
    3262             :   // Store both the high bits and maybe some of the low bits.
    3263          30 :   Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
    3264          10 :                          MMOFlags, AAInfo);
    3265             : 
    3266             :   // Increment the pointer to the other half.
    3267          10 :   Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
    3268             :   // Store the lowest ExcessBits bits in the second half.
    3269          20 :   Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
    3270          10 :                          N->getPointerInfo().getWithOffset(IncrementSize),
    3271          10 :                          EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
    3272          40 :                          MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    3273          20 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
    3274             : }
    3275             : 
    3276       97276 : SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
    3277       97276 :   SDValue InL, InH;
    3278      194552 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    3279             :   // Just truncate the low part of the source.
    3280      389104 :   return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
    3281             : }
    3282             : 
    3283          40 : SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
    3284          40 :   SDValue Op = N->getOperand(0);
    3285          80 :   EVT SrcVT = Op.getValueType();
    3286          80 :   EVT DstVT = N->getValueType(0);
    3287             :   SDLoc dl(N);
    3288             : 
    3289             :   // The following optimization is valid only if every value in SrcVT (when
    3290             :   // treated as signed) is representable in DstVT.  Check that the mantissa
    3291             :   // size of DstVT is >= than the number of bits in SrcVT -1.
    3292          40 :   const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
    3293          40 :   if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
    3294           0 :       TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){
    3295             :     // Do a signed conversion then adjust the result.
    3296           0 :     SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
    3297           0 :     SignedConv = TLI.LowerOperation(SignedConv, DAG);
    3298             : 
    3299             :     // The result of the signed conversion needs adjusting if the 'sign bit' of
    3300             :     // the incoming integer was set.  To handle this, we dynamically test to see
    3301             :     // if it is set, and, if so, add a fudge factor.
    3302             : 
    3303             :     const uint64_t F32TwoE32  = 0x4F800000ULL;
    3304             :     const uint64_t F32TwoE64  = 0x5F800000ULL;
    3305             :     const uint64_t F32TwoE128 = 0x7F800000ULL;
    3306             : 
    3307             :     APInt FF(32, 0);
    3308             :     if (SrcVT == MVT::i32)
    3309           0 :       FF = APInt(32, F32TwoE32);
    3310             :     else if (SrcVT == MVT::i64)
    3311           0 :       FF = APInt(32, F32TwoE64);
    3312             :     else if (SrcVT == MVT::i128)
    3313           0 :       FF = APInt(32, F32TwoE128);
    3314             :     else
    3315           0 :       llvm_unreachable("Unsupported UINT_TO_FP!");
    3316             : 
    3317             :     // Check whether the sign bit is set.
    3318           0 :     SDValue Lo, Hi;
    3319           0 :     GetExpandedInteger(Op, Lo, Hi);
    3320           0 :     SDValue SignSet = DAG.getSetCC(dl,
    3321             :                                    getSetCCResultType(Hi.getValueType()),
    3322             :                                    Hi,
    3323             :                                    DAG.getConstant(0, dl, Hi.getValueType()),
    3324           0 :                                    ISD::SETLT);
    3325             : 
    3326             :     // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
    3327             :     SDValue FudgePtr =
    3328           0 :         DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
    3329           0 :                             TLI.getPointerTy(DAG.getDataLayout()));
    3330             : 
    3331             :     // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
    3332           0 :     SDValue Zero = DAG.getIntPtrConstant(0, dl);
    3333           0 :     SDValue Four = DAG.getIntPtrConstant(4, dl);
    3334           0 :     if (DAG.getDataLayout().isBigEndian())
    3335             :       std::swap(Zero, Four);
    3336           0 :     SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
    3337           0 :                                    Zero, Four);
    3338           0 :     unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
    3339           0 :     FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
    3340           0 :                            FudgePtr, Offset);
    3341           0 :     Alignment = std::min(Alignment, 4u);
    3342             : 
    3343             :     // Load the value out, extending it from f32 to the destination float type.
    3344             :     // FIXME: Avoid the extend by constructing the right constant pool?
    3345           0 :     SDValue Fudge = DAG.getExtLoad(
    3346           0 :         ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
    3347             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
    3348           0 :         Alignment);
    3349           0 :     return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
    3350             :   }
    3351             : 
    3352             :   // Otherwise, use a libcall.
    3353          40 :   RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
    3354             :   assert(LC != RTLIB::UNKNOWN_LIBCALL &&
    3355             :          "Don't know how to expand this UINT_TO_FP!");
    3356          80 :   return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
    3357             : }
    3358             : 
    3359          13 : SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
    3360             :   SDLoc dl(N);
    3361          13 :   SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
    3362             :                                cast<AtomicSDNode>(N)->getMemoryVT(),
    3363             :                                N->getOperand(0),
    3364          13 :                                N->getOperand(1), N->getOperand(2),
    3365          26 :                                cast<AtomicSDNode>(N)->getMemOperand());
    3366          26 :   return Swap.getValue(1);
    3367             : }
    3368             : 
    3369             : 
    3370        2472 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
    3371        2472 :   SDValue InOp0 = N->getOperand(0);
    3372        2472 :   EVT InVT = InOp0.getValueType();
    3373             : 
    3374        4944 :   EVT OutVT = N->getValueType(0);
    3375        4944 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3376             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3377        2472 :   unsigned OutNumElems = OutVT.getVectorNumElements();
    3378        2472 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3379             : 
    3380             :   SDLoc dl(N);
    3381        2472 :   SDValue BaseIdx = N->getOperand(1);
    3382             : 
    3383             :   SmallVector<SDValue, 8> Ops;
    3384        2472 :   Ops.reserve(OutNumElems);
    3385       17548 :   for (unsigned i = 0; i != OutNumElems; ++i) {
    3386             : 
    3387             :     // Extract the element from the original vector.
    3388        7538 :     SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
    3389       15076 :       BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
    3390        7538 :     SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
    3391       15076 :       InVT.getVectorElementType(), N->getOperand(0), Index);
    3392             : 
    3393       15076 :     SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
    3394             :     // Insert the converted element to the new vector.
    3395        7538 :     Ops.push_back(Op);
    3396             :   }
    3397             : 
    3398        7416 :   return DAG.getBuildVector(NOutVT, dl, Ops);
    3399             : }
    3400             : 
    3401             : 
    3402         108 : SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
    3403             :   ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
    3404         216 :   EVT VT = N->getValueType(0);
    3405             :   SDLoc dl(N);
    3406             : 
    3407         108 :   ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
    3408             : 
    3409         216 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3410         216 :   SDValue V1 = GetPromotedInteger(N->getOperand(1));
    3411         216 :   EVT OutVT = V0.getValueType();
    3412             : 
    3413         216 :   return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
    3414             : }
    3415             : 
    3416             : 
    3417        1203 : SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
    3418        1203 :   EVT OutVT = N->getValueType(0);
    3419        2406 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3420             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3421        1203 :   unsigned NumElems = N->getNumOperands();
    3422        1203 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3423             : 
    3424             :   SDLoc dl(N);
    3425             : 
    3426             :   SmallVector<SDValue, 8> Ops;
    3427        1203 :   Ops.reserve(NumElems);
    3428       12039 :   for (unsigned i = 0; i != NumElems; ++i) {
    3429        5418 :     SDValue Op;
    3430             :     // BUILD_VECTOR integer operand types are allowed to be larger than the
    3431             :     // result's element type. This may still be true after the promotion. For
    3432             :     // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
    3433             :     // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
    3434       16254 :     if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
    3435       10572 :       Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
    3436             :     else
    3437         132 :       Op = N->getOperand(i);
    3438        5418 :     Ops.push_back(Op);
    3439             :   }
    3440             : 
    3441        3609 :   return DAG.getBuildVector(NOutVT, dl, Ops);
    3442             : }
    3443             : 
    3444          27 : SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
    3445             : 
    3446             :   SDLoc dl(N);
    3447             : 
    3448             :   assert(!N->getOperand(0).getValueType().isVector() &&
    3449             :          "Input must be a scalar");
    3450             : 
    3451          27 :   EVT OutVT = N->getValueType(0);
    3452          54 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3453             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3454          27 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3455             : 
    3456          81 :   SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
    3457             : 
    3458          81 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
    3459             : }
    3460             : 
    3461         177 : SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
    3462             :   SDLoc dl(N);
    3463             : 
    3464         177 :   EVT OutVT = N->getValueType(0);
    3465         354 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3466             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3467             : 
    3468         177 :   EVT OutElemTy = NOutVT.getVectorElementType();
    3469             : 
    3470         354 :   unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
    3471         177 :   unsigned NumOutElem = NOutVT.getVectorNumElements();
    3472         177 :   unsigned NumOperands = N->getNumOperands();
    3473             :   assert(NumElem * NumOperands == NumOutElem &&
    3474             :          "Unexpected number of elements");
    3475             : 
    3476             :   // Take the elements from the first vector.
    3477         354 :   SmallVector<SDValue, 8> Ops(NumOutElem);
    3478        1025 :   for (unsigned i = 0; i < NumOperands; ++i) {
    3479         848 :     SDValue Op = N->getOperand(i);
    3480         848 :     if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
    3481         424 :       Op = GetPromotedInteger(Op);
    3482         424 :     EVT SclrTy = Op.getValueType().getVectorElementType();
    3483             :     assert(NumElem == Op.getValueType().getVectorNumElements() &&
    3484             :            "Unexpected number of elements");
    3485             : 
    3486        2616 :     for (unsigned j = 0; j < NumElem; ++j) {
    3487        1096 :       SDValue Ext = DAG.getNode(
    3488             :           ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
    3489        3288 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3490        2192 :       Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
    3491             :     }
    3492             :   }
    3493             : 
    3494         531 :   return DAG.getBuildVector(NOutVT, dl, Ops);
    3495             : }
    3496             : 
    3497           3 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
    3498           3 :   EVT VT = N->getValueType(0);
    3499           6 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3500             :   assert(NVT.isVector() && "This type must be promoted to a vector type");
    3501             : 
    3502             :   SDLoc dl(N);
    3503             : 
    3504             :   // For operands whose TypeAction is to promote, extend the promoted node
    3505             :   // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
    3506             :   // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
    3507             :   // type..
    3508           9 :   if (getTypeAction(N->getOperand(0).getValueType())
    3509             :       == TargetLowering::TypePromoteInteger) {
    3510           3 :     SDValue Promoted;
    3511             : 
    3512           6 :     switch(N->getOpcode()) {
    3513           1 :       case ISD::SIGN_EXTEND_VECTOR_INREG:
    3514           2 :         Promoted = SExtPromotedInteger(N->getOperand(0));
    3515           1 :         break;
    3516           1 :       case ISD::ZERO_EXTEND_VECTOR_INREG:
    3517           2 :         Promoted = ZExtPromotedInteger(N->getOperand(0));
    3518           1 :         break;
    3519           1 :       case ISD::ANY_EXTEND_VECTOR_INREG:
    3520           2 :         Promoted = GetPromotedInteger(N->getOperand(0));
    3521           1 :         break;
    3522           0 :       default:
    3523           0 :         llvm_unreachable("Node has unexpected Opcode");
    3524             :     }
    3525           9 :     return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
    3526             :   }
    3527             : 
    3528             :   // Directly extend to the appropriate transform-to type.
    3529           0 :   return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
    3530             : }
    3531             : 
    3532         207 : SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
    3533         207 :   EVT OutVT = N->getValueType(0);
    3534         414 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3535             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3536             : 
    3537         207 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3538             : 
    3539             :   SDLoc dl(N);
    3540         414 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3541             : 
    3542         207 :   SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
    3543         414 :     NOutVTElem, N->getOperand(1));
    3544         207 :   return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
    3545         621 :     V0, ConvElem, N->getOperand(2));
    3546             : }
    3547             : 
    3548        5133 : SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    3549             :   SDLoc dl(N);
    3550       10266 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3551       10266 :   SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
    3552       15399 :                                   TLI.getVectorIdxTy(DAG.getDataLayout()));
    3553        5133 :   SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
    3554       10266 :     V0->getValueType(0).getScalarType(), V0, V1);
    3555             : 
    3556             :   // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
    3557             :   // element types. If this is the case then we need to expand the outgoing
    3558             :   // value and not truncate it.
    3559       15399 :   return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
    3560             : }
    3561             : 
    3562           0 : SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
    3563             :   SDLoc dl(N);
    3564           0 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3565           0 :   MVT InVT = V0.getValueType().getSimpleVT();
    3566             :   MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
    3567           0 :                                N->getValueType(0).getVectorNumElements());
    3568           0 :   SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
    3569           0 :   return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
    3570             : }
    3571             : 
    3572        2686 : SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
    3573             :   SDLoc dl(N);
    3574        2686 :   unsigned NumElems = N->getNumOperands();
    3575             : 
    3576        5372 :   EVT RetSclrTy = N->getValueType(0).getVectorElementType();
    3577             : 
    3578             :   SmallVector<SDValue, 8> NewOps;
    3579        2686 :   NewOps.reserve(NumElems);
    3580             : 
    3581             :   // For each incoming vector
    3582       16662 :   for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
    3583       13976 :     SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
    3584       13976 :     EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
    3585       13976 :     unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
    3586             : 
    3587       62932 :     for (unsigned i=0; i<NumElem; ++i) {
    3588             :       // Extract element from incoming vector
    3589       27972 :       SDValue Ex = DAG.getNode(
    3590             :           ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
    3591       83916 :           DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3592       55944 :       SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
    3593       27972 :       NewOps.push_back(Tr);
    3594             :     }
    3595             :   }
    3596             : 
    3597       10744 :   return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
    3598             : }

Generated by: LCOV version 1.13