LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeIntegerTypes.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1892 2058 91.9 %
Date: 2017-09-14 15:23:50 Functions: 122 131 93.1 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements integer type expansion and promotion for LegalizeTypes.
      11             : // Promotion is the act of changing a computation in an illegal type into a
      12             : // computation in a larger type.  For example, implementing i8 arithmetic in an
      13             : // i32 register (often needed on powerpc).
      14             : // Expansion is the act of changing a computation in an illegal type into a
      15             : // computation in two identical registers of a smaller type.  For example,
      16             : // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
      17             : // targets).
      18             : //
      19             : //===----------------------------------------------------------------------===//
      20             : 
      21             : #include "LegalizeTypes.h"
      22             : #include "llvm/IR/DerivedTypes.h"
      23             : #include "llvm/Support/ErrorHandling.h"
      24             : #include "llvm/Support/KnownBits.h"
      25             : #include "llvm/Support/raw_ostream.h"
      26             : using namespace llvm;
      27             : 
      28             : #define DEBUG_TYPE "legalize-types"
      29             : 
      30             : //===----------------------------------------------------------------------===//
      31             : //  Integer Result Promotion
      32             : //===----------------------------------------------------------------------===//
      33             : 
      34             : /// PromoteIntegerResult - This method is called when a result of a node is
      35             : /// found to be in need of promotion to a larger type.  At this point, the node
      36             : /// may also have invalid operands or may have other results that need
      37             : /// expansion, we just know that (at least) one result needs promotion.
      38      157569 : void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
      39             :   DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
      40      157569 :   SDValue Res = SDValue();
      41             : 
      42             :   // See if the target wants to custom expand this node.
      43      315138 :   if (CustomLowerNode(N, N->getValueType(ResNo), true))
      44         176 :     return;
      45             : 
      46      314786 :   switch (N->getOpcode()) {
      47           0 :   default:
      48             : #ifndef NDEBUG
      49             :     dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
      50             :     N->dump(&DAG); dbgs() << "\n";
      51             : #endif
      52           0 :     llvm_unreachable("Do not know how to promote this operator!");
      53           0 :   case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
      54          63 :   case ISD::AssertSext:  Res = PromoteIntRes_AssertSext(N); break;
      55          63 :   case ISD::AssertZext:  Res = PromoteIntRes_AssertZext(N); break;
      56        4983 :   case ISD::BITCAST:     Res = PromoteIntRes_BITCAST(N); break;
      57          15 :   case ISD::BITREVERSE:  Res = PromoteIntRes_BITREVERSE(N); break;
      58          18 :   case ISD::BSWAP:       Res = PromoteIntRes_BSWAP(N); break;
      59           0 :   case ISD::BUILD_PAIR:  Res = PromoteIntRes_BUILD_PAIR(N); break;
      60       15652 :   case ISD::Constant:    Res = PromoteIntRes_Constant(N); break;
      61           8 :   case ISD::CTLZ_ZERO_UNDEF:
      62           8 :   case ISD::CTLZ:        Res = PromoteIntRes_CTLZ(N); break;
      63          30 :   case ISD::CTPOP:       Res = PromoteIntRes_CTPOP(N); break;
      64          20 :   case ISD::CTTZ_ZERO_UNDEF:
      65          20 :   case ISD::CTTZ:        Res = PromoteIntRes_CTTZ(N); break;
      66       23649 :   case ISD::EXTRACT_VECTOR_ELT:
      67       23649 :                          Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
      68       12040 :   case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
      69           8 :   case ISD::MLOAD:       Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
      70           4 :     break;
      71          20 :   case ISD::MGATHER:     Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
      72          10 :     break;
      73         639 :   case ISD::SELECT:      Res = PromoteIntRes_SELECT(N); break;
      74          28 :   case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
      75           0 :   case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
      76       48582 :   case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
      77          92 :   case ISD::SMIN:
      78          92 :   case ISD::SMAX:        Res = PromoteIntRes_SExtIntBinOp(N); break;
      79          50 :   case ISD::UMIN:
      80          50 :   case ISD::UMAX:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
      81             : 
      82         752 :   case ISD::SHL:         Res = PromoteIntRes_SHL(N); break;
      83          53 :   case ISD::SIGN_EXTEND_INREG:
      84          53 :                          Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
      85          87 :   case ISD::SRA:         Res = PromoteIntRes_SRA(N); break;
      86        4799 :   case ISD::SRL:         Res = PromoteIntRes_SRL(N); break;
      87       27941 :   case ISD::TRUNCATE:    Res = PromoteIntRes_TRUNCATE(N); break;
      88        2101 :   case ISD::UNDEF:       Res = PromoteIntRes_UNDEF(N); break;
      89          36 :   case ISD::VAARG:       Res = PromoteIntRes_VAARG(N); break;
      90             : 
      91        2520 :   case ISD::EXTRACT_SUBVECTOR:
      92        2520 :                          Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
      93          84 :   case ISD::VECTOR_SHUFFLE:
      94          84 :                          Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
      95         182 :   case ISD::INSERT_VECTOR_ELT:
      96         182 :                          Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
      97        1644 :   case ISD::BUILD_VECTOR:
      98        1644 :                          Res = PromoteIntRes_BUILD_VECTOR(N); break;
      99          26 :   case ISD::SCALAR_TO_VECTOR:
     100          26 :                          Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
     101        1632 :   case ISD::CONCAT_VECTORS:
     102        1632 :                          Res = PromoteIntRes_CONCAT_VECTORS(N); break;
     103             : 
     104           3 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     105             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     106             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     107           3 :                          Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
     108             : 
     109        5061 :   case ISD::SIGN_EXTEND:
     110             :   case ISD::ZERO_EXTEND:
     111        5061 :   case ISD::ANY_EXTEND:  Res = PromoteIntRes_INT_EXTEND(N); break;
     112             : 
     113         141 :   case ISD::FP_TO_SINT:
     114         141 :   case ISD::FP_TO_UINT:  Res = PromoteIntRes_FP_TO_XINT(N); break;
     115             : 
     116         662 :   case ISD::FP_TO_FP16:  Res = PromoteIntRes_FP_TO_FP16(N); break;
     117             : 
     118        7297 :   case ISD::AND:
     119             :   case ISD::OR:
     120             :   case ISD::XOR:
     121             :   case ISD::ADD:
     122             :   case ISD::SUB:
     123        7297 :   case ISD::MUL:         Res = PromoteIntRes_SimpleIntBinOp(N); break;
     124             : 
     125         299 :   case ISD::SDIV:
     126         299 :   case ISD::SREM:        Res = PromoteIntRes_SExtIntBinOp(N); break;
     127             : 
     128         256 :   case ISD::UDIV:
     129         256 :   case ISD::UREM:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
     130             : 
     131          97 :   case ISD::SADDO:
     132          97 :   case ISD::SSUBO:       Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
     133         115 :   case ISD::UADDO:
     134         115 :   case ISD::USUBO:       Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
     135          83 :   case ISD::SMULO:
     136          83 :   case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;
     137             : 
     138           5 :   case ISD::ADDCARRY:
     139           5 :   case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
     140             : 
     141         101 :   case ISD::ATOMIC_LOAD:
     142         101 :     Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
     143             : 
     144         841 :   case ISD::ATOMIC_LOAD_ADD:
     145             :   case ISD::ATOMIC_LOAD_SUB:
     146             :   case ISD::ATOMIC_LOAD_AND:
     147             :   case ISD::ATOMIC_LOAD_OR:
     148             :   case ISD::ATOMIC_LOAD_XOR:
     149             :   case ISD::ATOMIC_LOAD_NAND:
     150             :   case ISD::ATOMIC_LOAD_MIN:
     151             :   case ISD::ATOMIC_LOAD_MAX:
     152             :   case ISD::ATOMIC_LOAD_UMIN:
     153             :   case ISD::ATOMIC_LOAD_UMAX:
     154             :   case ISD::ATOMIC_SWAP:
     155         841 :     Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
     156             : 
     157         649 :   case ISD::ATOMIC_CMP_SWAP:
     158             :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     159         649 :     Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
     160         649 :     break;
     161             :   }
     162             : 
     163             :   // If the result is null then the sub-method took care of registering it.
     164      157393 :   if (Res.getNode())
     165      157393 :     SetPromotedInteger(SDValue(N, ResNo), Res);
     166             : }
     167             : 
     168           0 : SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
     169             :                                                      unsigned ResNo) {
     170           0 :   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
     171           0 :   return GetPromotedInteger(Op);
     172             : }
     173             : 
     174          63 : SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
     175             :   // Sign-extend the new bits, and continue the assertion.
     176         126 :   SDValue Op = SExtPromotedInteger(N->getOperand(0));
     177         189 :   return DAG.getNode(ISD::AssertSext, SDLoc(N),
     178         315 :                      Op.getValueType(), Op, N->getOperand(1));
     179             : }
     180             : 
     181          63 : SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
     182             :   // Zero the new bits, and continue the assertion.
     183         126 :   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
     184         189 :   return DAG.getNode(ISD::AssertZext, SDLoc(N),
     185         315 :                      Op.getValueType(), Op, N->getOperand(1));
     186             : }
     187             : 
     188         101 : SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
     189         303 :   EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     190         303 :   SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
     191             :                               N->getMemoryVT(), ResVT,
     192         202 :                               N->getChain(), N->getBasePtr(),
     193         404 :                               N->getMemOperand());
     194             :   // Legalize the chain result - switch anything that used the old chain to
     195             :   // use the new one.
     196         303 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     197         101 :   return Res;
     198             : }
     199             : 
     200         841 : SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
     201        1682 :   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
     202        2523 :   SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
     203             :                               N->getMemoryVT(),
     204        1682 :                               N->getChain(), N->getBasePtr(),
     205        3364 :                               Op2, N->getMemOperand());
     206             :   // Legalize the chain result - switch anything that used the old chain to
     207             :   // use the new one.
     208        2523 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     209         841 :   return Res;
     210             : }
     211             : 
     212         649 : SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
     213             :                                                       unsigned ResNo) {
     214         649 :   if (ResNo == 1) {
     215             :     assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
     216        1566 :     EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
     217        1566 :     EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
     218             : 
     219             :     // Only use the result of getSetCCResultType if it is legal,
     220             :     // otherwise just use the promoted result type (NVT).
     221         522 :     if (!TLI.isTypeLegal(SVT))
     222           0 :       SVT = NVT;
     223             : 
     224        1566 :     SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
     225         522 :     SDValue Res = DAG.getAtomicCmpSwap(
     226        1044 :         ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
     227        2610 :         N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
     228        1566 :         N->getMemOperand());
     229        1566 :     ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
     230        1566 :     ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
     231        1044 :     return Res.getValue(1);
     232             :   }
     233             : 
     234         254 :   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
     235         254 :   SDValue Op3 = GetPromotedInteger(N->getOperand(3));
     236             :   SDVTList VTs =
     237         508 :       DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
     238         127 :   SDValue Res = DAG.getAtomicCmpSwap(
     239         508 :       N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
     240         635 :       N->getBasePtr(), Op2, Op3, N->getMemOperand());
     241             :   // Update the use to N with the newly created Res.
     242         508 :   for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
     243         762 :     ReplaceValueWith(SDValue(N, i), Res.getValue(i));
     244         127 :   return Res;
     245             : }
     246             : 
     247        4983 : SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
     248        9966 :   SDValue InOp = N->getOperand(0);
     249        9966 :   EVT InVT = InOp.getValueType();
     250        9966 :   EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
     251        9966 :   EVT OutVT = N->getValueType(0);
     252        9966 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
     253        9966 :   SDLoc dl(N);
     254             : 
     255        9966 :   switch (getTypeAction(InVT)) {
     256             :   case TargetLowering::TypeLegal:
     257             :     break;
     258         574 :   case TargetLowering::TypePromoteInteger:
     259         632 :     if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
     260             :       // The input promotes to the same size.  Convert the promoted value.
     261           0 :       return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
     262             :     break;
     263           0 :   case TargetLowering::TypeSoftenFloat:
     264             :     // Promote the integer operand by hand.
     265           0 :     return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
     266         432 :   case TargetLowering::TypePromoteFloat: {
     267             :     // Convert the promoted float by hand.
     268         432 :     SDValue PromotedOp = GetPromotedFloat(InOp);
     269         864 :     return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
     270             :     break;
     271             :   }
     272             :   case TargetLowering::TypeExpandInteger:
     273             :   case TargetLowering::TypeExpandFloat:
     274             :     break;
     275        2315 :   case TargetLowering::TypeScalarizeVector:
     276             :     // Convert the element to an integer and promote it by hand.
     277        2315 :     if (!NOutVT.isVector())
     278        2278 :       return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
     279        2278 :                          BitConvertToInteger(GetScalarizedVector(InOp)));
     280             :     break;
     281         457 :   case TargetLowering::TypeSplitVector: {
     282             :     // For example, i32 = BITCAST v2i16 on alpha.  Convert the split
     283             :     // pieces of the input into integers and reassemble in the final type.
     284         457 :     SDValue Lo, Hi;
     285         914 :     GetSplitVector(N->getOperand(0), Lo, Hi);
     286         457 :     Lo = BitConvertToInteger(Lo);
     287         457 :     Hi = BitConvertToInteger(Hi);
     288             : 
     289         914 :     if (DAG.getDataLayout().isBigEndian())
     290             :       std::swap(Lo, Hi);
     291             : 
     292         914 :     InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
     293         457 :                        EVT::getIntegerVT(*DAG.getContext(),
     294             :                                          NOutVT.getSizeInBits()),
     295        1371 :                        JoinIntegers(Lo, Hi));
     296         914 :     return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
     297             :   }
     298           2 :   case TargetLowering::TypeWidenVector:
     299             :     // The input is widened to the same size. Convert to the widened value.
     300             :     // Make sure that the outgoing value is not a vector, because this would
     301             :     // make us bitcast between two vectors which are legalized in different ways.
     302           4 :     if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
     303           0 :       return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
     304             :   }
     305             : 
     306        1816 :   return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
     307        1816 :                      CreateStackStoreLoad(InOp, OutVT));
     308             : }
     309             : 
     310          18 : SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
     311          36 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     312          36 :   EVT OVT = N->getValueType(0);
     313          36 :   EVT NVT = Op.getValueType();
     314          36 :   SDLoc dl(N);
     315             : 
     316          36 :   unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
     317          18 :   return DAG.getNode(
     318          18 :       ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
     319             :       DAG.getConstant(DiffBits, dl,
     320          72 :                       TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
     321             : }
     322             : 
     323          15 : SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
     324          30 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     325          30 :   EVT OVT = N->getValueType(0);
     326          30 :   EVT NVT = Op.getValueType();
     327          30 :   SDLoc dl(N);
     328             : 
     329          30 :   unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
     330          15 :   return DAG.getNode(
     331          15 :       ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
     332             :       DAG.getConstant(DiffBits, dl,
     333          60 :                       TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
     334             : }
     335             : 
     336           0 : SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
     337             :   // The pair element type may be legal, or may not promote to the same type as
     338             :   // the result, for example i14 = BUILD_PAIR (i7, i7).  Handle all cases.
     339           0 :   return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
     340           0 :                      TLI.getTypeToTransformTo(*DAG.getContext(),
     341           0 :                      N->getValueType(0)), JoinIntegers(N->getOperand(0),
     342           0 :                      N->getOperand(1)));
     343             : }
     344             : 
     345       15652 : SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
     346       31304 :   EVT VT = N->getValueType(0);
     347             :   // FIXME there is no actual debug info here
     348       31304 :   SDLoc dl(N);
     349             :   // Zero extend things like i1, sign extend everything else.  It shouldn't
     350             :   // matter in theory which one we pick, but this tends to give better code?
     351       15652 :   unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
     352       15652 :   SDValue Result = DAG.getNode(Opc, dl,
     353       31304 :                                TLI.getTypeToTransformTo(*DAG.getContext(), VT),
     354       62608 :                                SDValue(N, 0));
     355             :   assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
     356       31304 :   return Result;
     357             : }
     358             : 
     359           8 : SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
     360             :   // Zero extend to the promoted type and do the count there.
     361          16 :   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
     362          16 :   SDLoc dl(N);
     363          16 :   EVT OVT = N->getValueType(0);
     364          16 :   EVT NVT = Op.getValueType();
     365          24 :   Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
     366             :   // Subtract off the extra leading bits in the bigger type.
     367           8 :   return DAG.getNode(
     368             :       ISD::SUB, dl, NVT, Op,
     369          24 :       DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
     370          24 :                       NVT));
     371             : }
     372             : 
     373          30 : SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
     374             :   // Zero extend to the promoted type and do the count there.
     375          60 :   SDValue Op = ZExtPromotedInteger(N->getOperand(0));
     376         150 :   return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
     377             : }
     378             : 
     379          20 : SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
     380          40 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     381          40 :   EVT OVT = N->getValueType(0);
     382          40 :   EVT NVT = Op.getValueType();
     383          40 :   SDLoc dl(N);
     384          20 :   if (N->getOpcode() == ISD::CTTZ) {
     385             :     // The count is the same in the promoted type except if the original
     386             :     // value was zero.  This can be handled by setting the bit just off
     387             :     // the top of the original type.
     388             :     auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
     389          30 :                                       OVT.getScalarSizeInBits());
     390          20 :     Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
     391             :   }
     392          80 :   return DAG.getNode(N->getOpcode(), dl, NVT, Op);
     393             : }
     394             : 
     395       23649 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
     396       47298 :   SDLoc dl(N);
     397       70947 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     398       70947 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
     399       94596 :                      N->getOperand(1));
     400             : }
     401             : 
     402         141 : SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
     403         423 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     404         282 :   unsigned NewOpc = N->getOpcode();
     405         282 :   SDLoc dl(N);
     406             : 
     407             :   // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
     408             :   // not Legal, check to see if we can use FP_TO_SINT instead.  (If both UINT
     409             :   // and SINT conversions are Custom, there is no way to tell which is
     410             :   // preferable. We choose SINT because that's the right thing on PPC.)
     411         141 :   if (N->getOpcode() == ISD::FP_TO_UINT &&
     412         141 :       !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
     413          71 :       TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
     414             :     NewOpc = ISD::FP_TO_SINT;
     415             : 
     416         423 :   SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
     417             : 
     418             :   // Assert that the converted value fits in the original type.  If it doesn't
     419             :   // (eg: because the value being converted is too big), then the result of the
     420             :   // original operation was undefined anyway, so the assert is still correct.
     421             :   //
     422             :   // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
     423             :   //   before legalization: fp-to-uint16, 65534. -> 0xfffe
     424             :   //   after legalization: fp-to-sint32, 65534. -> 0x0000fffe
     425         282 :   return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
     426             :                      ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
     427         564 :                      DAG.getValueType(N->getValueType(0).getScalarType()));
     428             : }
     429             : 
     430         662 : SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
     431        1986 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     432        1324 :   SDLoc dl(N);
     433             : 
     434        3310 :   return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
     435             : }
     436             : 
     437        5061 : SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
     438       15183 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     439       10122 :   SDLoc dl(N);
     440             : 
     441       20244 :   if (getTypeAction(N->getOperand(0).getValueType())
     442             :       == TargetLowering::TypePromoteInteger) {
     443        8588 :     SDValue Res = GetPromotedInteger(N->getOperand(0));
     444             :     assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
     445             : 
     446             :     // If the result and operand types are the same after promotion, simplify
     447             :     // to an in-register extension.
     448        8592 :     if (NVT == Res.getValueType()) {
     449             :       // The high bits are not guaranteed to be anything.  Insert an extend.
     450        8574 :       if (N->getOpcode() == ISD::SIGN_EXTEND)
     451         421 :         return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
     452        1263 :                            DAG.getValueType(N->getOperand(0).getValueType()));
     453        3866 :       if (N->getOpcode() == ISD::ZERO_EXTEND)
     454         693 :         return DAG.getZeroExtendInReg(Res, dl,
     455        2079 :                       N->getOperand(0).getValueType().getScalarType());
     456             :       assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
     457        3173 :       return Res;
     458             :     }
     459             :   }
     460             : 
     461             :   // Otherwise, just extend the original operand all the way to the larger type.
     462        3096 :   return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
     463             : }
     464             : 
     465        6020 : SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
     466             :   assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
     467       18060 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     468             :   ISD::LoadExtType ExtType =
     469         294 :     ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
     470       12040 :   SDLoc dl(N);
     471       18060 :   SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
     472       12040 :                                N->getMemoryVT(), N->getMemOperand());
     473             : 
     474             :   // Legalize the chain result - switch anything that used the old chain to
     475             :   // use the new one.
     476       18060 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     477       12040 :   return Res;
     478             : }
     479             : 
     480           4 : SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
     481          12 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     482           4 :   SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
     483             : 
     484           8 :   SDLoc dl(N);
     485          16 :   SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
     486           8 :                                   N->getMask(), ExtSrc0, N->getMemoryVT(),
     487           8 :                                   N->getMemOperand(), ISD::SEXTLOAD);
     488             :   // Legalize the chain result - switch anything that used the old chain to
     489             :   // use the new one.
     490          12 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     491           8 :   return Res;
     492             : }
     493             : 
     494          10 : SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
     495          30 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     496          20 :   SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
     497             :   assert(NVT == ExtSrc0.getValueType() &&
     498             :       "Gather result type and the passThru agrument type should be the same");
     499             : 
     500          20 :   SDLoc dl(N);
     501          60 :   SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
     502          40 :                    N->getIndex()};
     503          30 :   SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
     504             :                                     N->getMemoryVT(), dl, Ops,
     505          30 :                                     N->getMemOperand());
     506             :   // Legalize the chain result - switch anything that used the old chain to
     507             :   // use the new one.
     508          30 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     509          20 :   return Res;
     510             : }
     511             : 
     512             : /// Promote the overflow flag of an overflowing arithmetic node.
     513         292 : SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
     514             :   // Simply change the return type of the boolean result.
     515         876 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
     516         584 :   EVT ValueVTs[] = { N->getValueType(0), NVT };
     517         876 :   SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
     518         584 :   unsigned NumOps = N->getNumOperands();
     519             :   assert(NumOps <= 3 && "Too many operands");
     520         292 :   if (NumOps == 3)
     521          10 :     Ops[2] = N->getOperand(2);
     522             : 
     523         876 :   SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
     524        1752 :                             DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
     525             : 
     526             :   // Modified the sum result - switch anything that used the old sum to use
     527             :   // the new one.
     528         292 :   ReplaceValueWith(SDValue(N, 0), Res);
     529             : 
     530         292 :   return SDValue(Res.getNode(), 1);
     531             : }
     532             : 
     533          97 : SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
     534          97 :   if (ResNo == 1)
     535          95 :     return PromoteIntRes_Overflow(N);
     536             : 
     537             :   // The operation overflowed iff the result in the larger type is not the
     538             :   // sign extension of its truncation to the original type.
     539           4 :   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
     540           4 :   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
     541           6 :   EVT OVT = N->getOperand(0).getValueType();
     542           4 :   EVT NVT = LHS.getValueType();
     543           2 :   SDLoc dl(N);
     544             : 
     545             :   // Do the arithmetic in the larger type.
     546           2 :   unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
     547           4 :   SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
     548             : 
     549             :   // Calculate the overflow flag: sign extend the arithmetic result from
     550             :   // the original type.
     551           2 :   SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
     552           2 :                             DAG.getValueType(OVT));
     553             :   // Overflowed if and only if this is not equal to Res.
     554           4 :   Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
     555             : 
     556             :   // Use the calculated overflow everywhere.
     557           2 :   ReplaceValueWith(SDValue(N, 1), Ofl);
     558             : 
     559           2 :   return Res;
     560             : }
     561             : 
     562         639 : SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
     563        1278 :   SDValue LHS = GetPromotedInteger(N->getOperand(1));
     564        1278 :   SDValue RHS = GetPromotedInteger(N->getOperand(2));
     565        1917 :   return DAG.getSelect(SDLoc(N),
     566        3195 :                        LHS.getValueType(), N->getOperand(0), LHS, RHS);
     567             : }
     568             : 
     569          28 : SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
     570          56 :   SDValue Mask = N->getOperand(0);
     571          84 :   EVT OpTy = N->getOperand(1).getValueType();
     572             : 
     573             :   // Promote all the way up to the canonical SetCC type.
     574          28 :   Mask = PromoteTargetBoolean(Mask, OpTy);
     575          56 :   SDValue LHS = GetPromotedInteger(N->getOperand(1));
     576          56 :   SDValue RHS = GetPromotedInteger(N->getOperand(2));
     577          84 :   return DAG.getNode(ISD::VSELECT, SDLoc(N),
     578         112 :                      LHS.getValueType(), Mask, LHS, RHS);
     579             : }
     580             : 
     581           0 : SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
     582           0 :   SDValue LHS = GetPromotedInteger(N->getOperand(2));
     583           0 :   SDValue RHS = GetPromotedInteger(N->getOperand(3));
     584           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
     585           0 :                      LHS.getValueType(), N->getOperand(0),
     586           0 :                      N->getOperand(1), LHS, RHS, N->getOperand(4));
     587             : }
     588             : 
     589       48582 : SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
     590      145746 :   EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
     591             : 
     592      145746 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     593             : 
     594             :   // Only use the result of getSetCCResultType if it is legal,
     595             :   // otherwise just use the promoted result type (NVT).
     596       48582 :   if (!TLI.isTypeLegal(SVT))
     597        1364 :     SVT = NVT;
     598             : 
     599       97164 :   SDLoc dl(N);
     600             :   assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
     601             :          "Vector compare must return a vector result!");
     602             : 
     603       97164 :   SDValue LHS = N->getOperand(0);
     604       97164 :   SDValue RHS = N->getOperand(1);
     605      145808 :   if (LHS.getValueType() != RHS.getValueType()) {
     606           3 :     if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger &&
     607           1 :         !LHS.getValueType().isVector())
     608           0 :       LHS = GetPromotedInteger(LHS);
     609           4 :     if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
     610           3 :         !RHS.getValueType().isVector())
     611           1 :       RHS = GetPromotedInteger(RHS);
     612             :   }
     613             : 
     614             :   // Get the SETCC result using the canonical SETCC type.
     615       48582 :   SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
     616      145746 :                               N->getOperand(2));
     617             : 
     618             :   // Convert to the expected type.
     619       97164 :   return DAG.getSExtOrTrunc(SetCC, dl, NVT);
     620             : }
     621             : 
     622         752 : SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
     623        1504 :   SDValue LHS = N->getOperand(0);
     624        1504 :   SDValue RHS = N->getOperand(1);
     625        2256 :   if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
     626         752 :     LHS = GetPromotedInteger(LHS);
     627        2256 :   if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
     628          51 :     RHS = ZExtPromotedInteger(RHS);
     629        3760 :   return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
     630             : }
     631             : 
     632          53 : SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
     633         106 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
     634         159 :   return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
     635         265 :                      Op.getValueType(), Op, N->getOperand(1));
     636             : }
     637             : 
     638        7297 : SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
     639             :   // The input may have strange things in the top bits of the registers, but
     640             :   // these operations don't care.  They may have weird bits going out, but
     641             :   // that too is okay if they are integer operations.
     642       14594 :   SDValue LHS = GetPromotedInteger(N->getOperand(0));
     643       14594 :   SDValue RHS = GetPromotedInteger(N->getOperand(1));
     644       21891 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     645       36485 :                      LHS.getValueType(), LHS, RHS);
     646             : }
     647             : 
     648         391 : SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
     649             :   // Sign extend the input.
     650         782 :   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
     651         782 :   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
     652        1173 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     653        1955 :                      LHS.getValueType(), LHS, RHS);
     654             : }
     655             : 
     656         306 : SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
     657             :   // Zero extend the input.
     658         612 :   SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
     659         612 :   SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
     660         918 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     661        1530 :                      LHS.getValueType(), LHS, RHS);
     662             : }
     663             : 
     664          87 : SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
     665         174 :   SDValue LHS = N->getOperand(0);
     666         174 :   SDValue RHS = N->getOperand(1);
     667             :   // The input value must be properly sign extended.
     668         261 :   if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
     669          87 :     LHS = SExtPromotedInteger(LHS);
     670         261 :   if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
     671          42 :     RHS = ZExtPromotedInteger(RHS);
     672         435 :   return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
     673             : }
     674             : 
     675        4799 : SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
     676        9598 :   SDValue LHS = N->getOperand(0);
     677        9598 :   SDValue RHS = N->getOperand(1);
     678             :   // The input value must be properly zero extended.
     679       14397 :   if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
     680        4799 :     LHS = ZExtPromotedInteger(LHS);
     681       14397 :   if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
     682          56 :     RHS = ZExtPromotedInteger(RHS);
     683       23995 :   return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
     684             : }
     685             : 
     686       27941 : SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
     687       83823 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     688       27941 :   SDValue Res;
     689       55882 :   SDValue InOp = N->getOperand(0);
     690       55882 :   SDLoc dl(N);
     691             : 
     692       83823 :   switch (getTypeAction(InOp.getValueType())) {
     693           0 :   default: llvm_unreachable("Unknown type action!");
     694       23473 :   case TargetLowering::TypeLegal:
     695             :   case TargetLowering::TypeExpandInteger:
     696       23473 :     Res = InOp;
     697       23473 :     break;
     698        4454 :   case TargetLowering::TypePromoteInteger:
     699        4454 :     Res = GetPromotedInteger(InOp);
     700        4454 :     break;
     701          13 :   case TargetLowering::TypeSplitVector: {
     702          26 :     EVT InVT = InOp.getValueType();
     703             :     assert(InVT.isVector() && "Cannot split scalar types");
     704          13 :     unsigned NumElts = InVT.getVectorNumElements();
     705             :     assert(NumElts == NVT.getVectorNumElements() &&
     706             :            "Dst and Src must have the same number of elements");
     707             :     assert(isPowerOf2_32(NumElts) &&
     708             :            "Promoted vector type must be a power of two");
     709             : 
     710          13 :     SDValue EOp1, EOp2;
     711          13 :     GetSplitVector(InOp, EOp1, EOp2);
     712             : 
     713          13 :     EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
     714          26 :                                    NumElts/2);
     715          26 :     EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
     716          26 :     EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
     717             : 
     718          26 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
     719             :   }
     720           1 :   case TargetLowering::TypeWidenVector: {
     721           1 :     SDValue WideInOp = GetWidenedVector(InOp);
     722             : 
     723             :     // Truncate widened InOp.
     724           2 :     unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
     725           1 :     EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
     726           3 :                                    N->getValueType(0).getScalarType(), NumElem);
     727           2 :     SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
     728             : 
     729             :     // Zero extend so that the elements are of same type as those of NVT
     730           1 :     EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
     731           2 :                                  NumElem);
     732           2 :     SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
     733             : 
     734             :     // Extract the low NVT subvector.
     735           2 :     MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
     736           2 :     SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
     737           2 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
     738             :   }
     739             :   }
     740             : 
     741             :   // Truncate to NVT instead of VT
     742       55854 :   return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
     743             : }
     744             : 
     745         115 : SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
     746         115 :   if (ResNo == 1)
     747         110 :     return PromoteIntRes_Overflow(N);
     748             : 
     749             :   // The operation overflowed iff the result in the larger type is not the
     750             :   // zero extension of its truncation to the original type.
     751          10 :   SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
     752          10 :   SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
     753          15 :   EVT OVT = N->getOperand(0).getValueType();
     754          10 :   EVT NVT = LHS.getValueType();
     755           5 :   SDLoc dl(N);
     756             : 
     757             :   // Do the arithmetic in the larger type.
     758           5 :   unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
     759          10 :   SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
     760             : 
     761             :   // Calculate the overflow flag: zero extend the arithmetic result from
     762             :   // the original type.
     763           5 :   SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
     764             :   // Overflowed if and only if this is not equal to Res.
     765          10 :   Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
     766             : 
     767             :   // Use the calculated overflow everywhere.
     768           5 :   ReplaceValueWith(SDValue(N, 1), Ofl);
     769             : 
     770           5 :   return Res;
     771             : }
     772             : 
     773           0 : SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
     774           5 :   if (ResNo == 1)
     775           5 :     return PromoteIntRes_Overflow(N);
     776           0 :   llvm_unreachable("Not implemented");
     777             : }
     778             : 
     779          83 : SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
     780             :   // Promote the overflow bit trivially.
     781          83 :   if (ResNo == 1)
     782          82 :     return PromoteIntRes_Overflow(N);
     783             : 
     784           3 :   SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
     785           1 :   SDLoc DL(N);
     786           2 :   EVT SmallVT = LHS.getValueType();
     787             : 
     788             :   // To determine if the result overflowed in a larger type, we extend the
     789             :   // input to the larger type, do the multiply (checking if it overflows),
     790             :   // then also check the high bits of the result to see if overflow happened
     791             :   // there.
     792           1 :   if (N->getOpcode() == ISD::SMULO) {
     793           1 :     LHS = SExtPromotedInteger(LHS);
     794           1 :     RHS = SExtPromotedInteger(RHS);
     795             :   } else {
     796           0 :     LHS = ZExtPromotedInteger(LHS);
     797           0 :     RHS = ZExtPromotedInteger(RHS);
     798             :   }
     799           3 :   SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
     800           2 :   SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
     801             : 
     802             :   // Overflow occurred if it occurred in the larger type, or if the high part
     803             :   // of the result does not zero/sign-extend the low part.  Check this second
     804             :   // possibility first.
     805           1 :   SDValue Overflow;
     806           1 :   if (N->getOpcode() == ISD::UMULO) {
     807             :     // Unsigned overflow occurred if the high part is non-zero.
     808           0 :     SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
     809           0 :                              DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
     810           0 :                                                    DL));
     811           0 :     Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
     812             :                             DAG.getConstant(0, DL, Hi.getValueType()),
     813           0 :                             ISD::SETNE);
     814             :   } else {
     815             :     // Signed overflow occurred if the high part does not sign extend the low.
     816           1 :     SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
     817           2 :                                Mul, DAG.getValueType(SmallVT));
     818           2 :     Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
     819             :   }
     820             : 
     821             :   // The only other way for overflow to occur is if the multiplication in the
     822             :   // larger type itself overflowed.
     823           2 :   Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
     824           4 :                          SDValue(Mul.getNode(), 1));
     825             : 
     826             :   // Use the calculated overflow everywhere.
     827           1 :   ReplaceValueWith(SDValue(N, 1), Overflow);
     828           1 :   return Mul;
     829             : }
     830             : 
     831        2101 : SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
     832        4202 :   return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
     833        6303 :                                                N->getValueType(0)));
     834             : }
     835             : 
     836          36 : SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
     837          72 :   SDValue Chain = N->getOperand(0); // Get the chain.
     838          72 :   SDValue Ptr = N->getOperand(1); // Get the pointer.
     839          72 :   EVT VT = N->getValueType(0);
     840          72 :   SDLoc dl(N);
     841             : 
     842          36 :   MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
     843          36 :   unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
     844             :   // The argument is passed as NumRegs registers of type RegVT.
     845             : 
     846         108 :   SmallVector<SDValue, 8> Parts(NumRegs);
     847          72 :   for (unsigned i = 0; i < NumRegs; ++i) {
     848         180 :     Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
     849         108 :                             N->getConstantOperandVal(3));
     850         108 :     Chain = Parts[i].getValue(1);
     851             :   }
     852             : 
     853             :   // Handle endianness of the load.
     854          72 :   if (DAG.getDataLayout().isBigEndian())
     855          36 :     std::reverse(Parts.begin(), Parts.end());
     856             : 
     857             :   // Assemble the parts in the promoted type.
     858         108 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
     859         108 :   SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
     860          36 :   for (unsigned i = 1; i < NumRegs; ++i) {
     861           0 :     SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
     862             :     // Shift it to the right position and "or" it in.
     863           0 :     Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
     864           0 :                        DAG.getConstant(i * RegVT.getSizeInBits(), dl,
     865           0 :                                        TLI.getPointerTy(DAG.getDataLayout())));
     866           0 :     Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
     867             :   }
     868             : 
     869             :   // Modified the chain result - switch anything that used the old chain to
     870             :   // use the new one.
     871          36 :   ReplaceValueWith(SDValue(N, 1), Chain);
     872             : 
     873          72 :   return Res;
     874             : }
     875             : 
     876             : //===----------------------------------------------------------------------===//
     877             : //  Integer Operand Promotion
     878             : //===----------------------------------------------------------------------===//
     879             : 
     880             : /// PromoteIntegerOperand - This method is called when the specified operand of
     881             : /// the specified node is found to need promotion.  At this point, all of the
     882             : /// result types of the node are known to be legal, but other operands of the
     883             : /// node may need promotion or expansion as well as the specified one.
     884      176323 : bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
     885             :   DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
     886      176323 :   SDValue Res = SDValue();
     887             : 
     888      528969 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
     889             :     return false;
     890             : 
     891      351876 :   switch (N->getOpcode()) {
     892           0 :     default:
     893             :   #ifndef NDEBUG
     894             :     dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
     895             :     N->dump(&DAG); dbgs() << "\n";
     896             :   #endif
     897           0 :     llvm_unreachable("Do not know how to promote this operator's operand!");
     898             : 
     899       23583 :   case ISD::ANY_EXTEND:   Res = PromoteIntOp_ANY_EXTEND(N); break;
     900         103 :   case ISD::ATOMIC_STORE:
     901         103 :     Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
     902         103 :     break;
     903        2526 :   case ISD::BITCAST:      Res = PromoteIntOp_BITCAST(N); break;
     904          12 :   case ISD::BR_CC:        Res = PromoteIntOp_BR_CC(N, OpNo); break;
     905       33694 :   case ISD::BRCOND:       Res = PromoteIntOp_BRCOND(N, OpNo); break;
     906           0 :   case ISD::BUILD_PAIR:   Res = PromoteIntOp_BUILD_PAIR(N); break;
     907        5741 :   case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
     908        1678 :   case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
     909       14880 :   case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
     910         173 :   case ISD::INSERT_VECTOR_ELT:
     911         173 :                           Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
     912          21 :   case ISD::SCALAR_TO_VECTOR:
     913          21 :                           Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
     914        4115 :   case ISD::VSELECT:
     915        4115 :   case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
     916         104 :   case ISD::SELECT_CC:    Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
     917        4557 :   case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
     918        7999 :   case ISD::SIGN_EXTEND:  Res = PromoteIntOp_SIGN_EXTEND(N); break;
     919          84 :   case ISD::SINT_TO_FP:   Res = PromoteIntOp_SINT_TO_FP(N); break;
     920       12956 :   case ISD::STORE:        Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
     921       12956 :                                                    OpNo); break;
     922          44 :   case ISD::MSTORE:       Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
     923          44 :                                                     OpNo); break;
     924          90 :   case ISD::MLOAD:        Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
     925          90 :                                                     OpNo); break;
     926          84 :   case ISD::MGATHER:  Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
     927          84 :                                                  OpNo); break;
     928          44 :   case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
     929          44 :                                                   OpNo); break;
     930        2819 :   case ISD::TRUNCATE:     Res = PromoteIntOp_TRUNCATE(N); break;
     931        1559 :   case ISD::FP16_TO_FP:
     932        1559 :   case ISD::UINT_TO_FP:   Res = PromoteIntOp_UINT_TO_FP(N); break;
     933       66897 :   case ISD::ZERO_EXTEND:  Res = PromoteIntOp_ZERO_EXTEND(N); break;
     934          42 :   case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
     935             : 
     936           0 :   case ISD::SHL:
     937             :   case ISD::SRA:
     938             :   case ISD::SRL:
     939             :   case ISD::ROTL:
     940           0 :   case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
     941             : 
     942           5 :   case ISD::ADDCARRY:
     943           5 :   case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
     944             :   }
     945             : 
     946             :   // If the result is null, the sub-method took care of registering results etc.
     947      175938 :   if (!Res.getNode()) return false;
     948             : 
     949             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
     950             :   // core about this.
     951      175936 :   if (Res.getNode() == N)
     952             :     return true;
     953             : 
     954             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
     955             :          "Invalid operand expansion");
     956             : 
     957      126762 :   ReplaceValueWith(SDValue(N, 0), Res);
     958      126762 :   return false;
     959             : }
     960             : 
     961             : /// PromoteSetCCOperands - Promote the operands of a comparison.  This code is
     962             : /// shared among BR_CC, SELECT_CC, and SETCC handlers.
     963        4673 : void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
     964             :                                             ISD::CondCode CCCode) {
     965             :   // We have to insert explicit sign or zero extends.  Note that we could
     966             :   // insert sign extends for ALL conditions, but zero extend is cheaper on
     967             :   // many machines (an AND instead of two shifts), so prefer it.
     968             :   switch (CCCode) {
     969           0 :   default: llvm_unreachable("Unknown integer comparison!");
     970        4204 :   case ISD::SETEQ:
     971             :   case ISD::SETNE: {
     972        4204 :     SDValue OpL = GetPromotedInteger(NewLHS);
     973        4204 :     SDValue OpR = GetPromotedInteger(NewRHS);
     974             : 
     975             :     // We would prefer to promote the comparison operand with sign extension.
     976             :     // If the width of OpL/OpR excluding the duplicated sign bits is no greater
     977             :     // than the width of NewLHS/NewRH, we can avoid inserting real truncate
     978             :     // instruction, which is redudant eventually.
     979             :     unsigned OpLEffectiveBits =
     980        4204 :         OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
     981             :     unsigned OpREffectiveBits =
     982        4204 :         OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
     983        4374 :     if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() &&
     984         170 :         OpREffectiveBits <= NewRHS.getValueSizeInBits()) {
     985         167 :       NewLHS = OpL;
     986         167 :       NewRHS = OpR;
     987             :     } else {
     988        4037 :       NewLHS = ZExtPromotedInteger(NewLHS);
     989        4037 :       NewRHS = ZExtPromotedInteger(NewRHS);
     990             :     }
     991             :     break;
     992             :   }
     993         192 :   case ISD::SETUGE:
     994             :   case ISD::SETUGT:
     995             :   case ISD::SETULE:
     996             :   case ISD::SETULT:
     997             :     // ALL of these operations will work if we either sign or zero extend
     998             :     // the operands (including the unsigned comparisons!).  Zero extend is
     999             :     // usually a simpler/cheaper operation, so prefer it.
    1000         192 :     NewLHS = ZExtPromotedInteger(NewLHS);
    1001         192 :     NewRHS = ZExtPromotedInteger(NewRHS);
    1002         192 :     break;
    1003         277 :   case ISD::SETGE:
    1004             :   case ISD::SETGT:
    1005             :   case ISD::SETLT:
    1006             :   case ISD::SETLE:
    1007         277 :     NewLHS = SExtPromotedInteger(NewLHS);
    1008         277 :     NewRHS = SExtPromotedInteger(NewRHS);
    1009         277 :     break;
    1010             :   }
    1011        4673 : }
    1012             : 
    1013       23583 : SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
    1014       47166 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1015      117915 :   return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
    1016             : }
    1017             : 
    1018         103 : SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
    1019         206 :   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
    1020         309 :   return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
    1021         618 :                        N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
    1022             : }
    1023             : 
    1024           0 : SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
    1025             :   // This should only occur in unusual situations like bitcasting to an
    1026             :   // x86_fp80, so just turn it into a store+load
    1027        3789 :   return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
    1028             : }
    1029             : 
    1030          12 : SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
    1031             :   assert(OpNo == 2 && "Don't know how to promote this operand!");
    1032             : 
    1033          24 :   SDValue LHS = N->getOperand(2);
    1034          24 :   SDValue RHS = N->getOperand(3);
    1035          36 :   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
    1036             : 
    1037             :   // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
    1038             :   // legal types.
    1039          36 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1040          36 :                                 N->getOperand(1), LHS, RHS, N->getOperand(4)),
    1041          12 :                  0);
    1042             : }
    1043             : 
    1044       33694 : SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
    1045             :   assert(OpNo == 1 && "only know how to promote condition");
    1046             : 
    1047             :   // Promote all the way up to the canonical SetCC type.
    1048       67388 :   SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
    1049             : 
    1050             :   // The chain (Op#0) and basic block destination (Op#2) are always legal types.
    1051      101082 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
    1052      101082 :                                         N->getOperand(2)), 0);
    1053             : }
    1054             : 
    1055           0 : SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
    1056             :   // Since the result type is legal, the operands must promote to it.
    1057           0 :   EVT OVT = N->getOperand(0).getValueType();
    1058           0 :   SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
    1059           0 :   SDValue Hi = GetPromotedInteger(N->getOperand(1));
    1060             :   assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
    1061           0 :   SDLoc dl(N);
    1062             : 
    1063           0 :   Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
    1064           0 :                    DAG.getConstant(OVT.getSizeInBits(), dl,
    1065           0 :                                    TLI.getPointerTy(DAG.getDataLayout())));
    1066           0 :   return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
    1067             : }
    1068             : 
    1069        5741 : SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
    1070             :   // The vector type is legal but the element type is not.  This implies
    1071             :   // that the vector is a power-of-two in length and that the element
    1072             :   // type does not have a strange size (eg: it is not i1).
    1073       11482 :   EVT VecVT = N->getValueType(0);
    1074        5741 :   unsigned NumElts = VecVT.getVectorNumElements();
    1075             :   assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
    1076             :          "Legal vector of one illegal element?");
    1077             : 
    1078             :   // Promote the inserted value.  The type does not need to match the
    1079             :   // vector element type.  Check that any extra bits introduced will be
    1080             :   // truncated away.
    1081             :   assert(N->getOperand(0).getValueSizeInBits() >=
    1082             :          N->getValueType(0).getScalarSizeInBits() &&
    1083             :          "Type of inserted value narrower than vector element type!");
    1084             : 
    1085       11482 :   SmallVector<SDValue, 16> NewOps;
    1086       65161 :   for (unsigned i = 0; i < NumElts; ++i)
    1087      118840 :     NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
    1088             : 
    1089       17223 :   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1090             : }
    1091             : 
    1092         173 : SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
    1093             :                                                          unsigned OpNo) {
    1094         173 :   if (OpNo == 1) {
    1095             :     // Promote the inserted value.  This is valid because the type does not
    1096             :     // have to match the vector element type.
    1097             : 
    1098             :     // Check that any extra bits introduced will be truncated away.
    1099             :     assert(N->getOperand(1).getValueSizeInBits() >=
    1100             :            N->getValueType(0).getScalarSizeInBits() &&
    1101             :            "Type of inserted value narrower than vector element type!");
    1102         519 :     return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1103         346 :                                   GetPromotedInteger(N->getOperand(1)),
    1104         346 :                                   N->getOperand(2)),
    1105         173 :                    0);
    1106             :   }
    1107             : 
    1108             :   assert(OpNo == 2 && "Different operand and result vector types?");
    1109             : 
    1110             :   // Promote the index.
    1111           0 :   SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
    1112           0 :                                    TLI.getVectorIdxTy(DAG.getDataLayout()));
    1113           0 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1114           0 :                                 N->getOperand(1), Idx), 0);
    1115             : }
    1116             : 
    1117          21 : SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
    1118             :   // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
    1119             :   // the operand in place.
    1120          42 :   return SDValue(DAG.UpdateNodeOperands(N,
    1121          63 :                                 GetPromotedInteger(N->getOperand(0))), 0);
    1122             : }
    1123             : 
    1124        4115 : SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
    1125             :   assert(OpNo == 0 && "Only know how to promote the condition!");
    1126        8230 :   SDValue Cond = N->getOperand(0);
    1127       12345 :   EVT OpTy = N->getOperand(1).getValueType();
    1128             : 
    1129        4115 :   if (N->getOpcode() == ISD::VSELECT)
    1130        1378 :     if (SDValue Res = WidenVSELECTAndMask(N))
    1131         968 :       return Res;
    1132             : 
    1133             :   // Promote all the way up to the canonical SetCC type.
    1134        3147 :   EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
    1135        3147 :   Cond = PromoteTargetBoolean(Cond, OpVT);
    1136             : 
    1137        9441 :   return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
    1138        9441 :                                         N->getOperand(2)), 0);
    1139             : }
    1140             : 
    1141         104 : SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
    1142             :   assert(OpNo == 0 && "Don't know how to promote this operand!");
    1143             : 
    1144         208 :   SDValue LHS = N->getOperand(0);
    1145         208 :   SDValue RHS = N->getOperand(1);
    1146         312 :   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
    1147             : 
    1148             :   // The CC (#4) and the possible return values (#2 and #3) have legal types.
    1149         312 :   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
    1150         416 :                                 N->getOperand(3), N->getOperand(4)), 0);
    1151             : }
    1152             : 
    1153        4557 : SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
    1154             :   assert(OpNo == 0 && "Don't know how to promote this operand!");
    1155             : 
    1156        9114 :   SDValue LHS = N->getOperand(0);
    1157        9114 :   SDValue RHS = N->getOperand(1);
    1158       13671 :   PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
    1159             : 
    1160             :   // The CC (#2) is always legal.
    1161        9114 :   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
    1162             : }
    1163             : 
    1164           0 : SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
    1165           0 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    1166           0 :                                 ZExtPromotedInteger(N->getOperand(1))), 0);
    1167             : }
    1168             : 
    1169        7999 : SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
    1170       15998 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1171       15998 :   SDLoc dl(N);
    1172       23997 :   Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
    1173        7999 :   return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
    1174       39995 :                      Op, DAG.getValueType(N->getOperand(0).getValueType()));
    1175             : }
    1176             : 
    1177          84 : SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
    1178         168 :   return SDValue(DAG.UpdateNodeOperands(N,
    1179         252 :                                 SExtPromotedInteger(N->getOperand(0))), 0);
    1180             : }
    1181             : 
    1182        6478 : SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
    1183             :   assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
    1184       19434 :   SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
    1185       12956 :   SDLoc dl(N);
    1186             : 
    1187        6478 :   SDValue Val = GetPromotedInteger(N->getValue());  // Get promoted value.
    1188             : 
    1189             :   // Truncate the value and store the result.
    1190        6478 :   return DAG.getTruncStore(Ch, dl, Val, Ptr,
    1191       12956 :                            N->getMemoryVT(), N->getMemOperand());
    1192             : }
    1193             : 
    1194          25 : SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
    1195             :                                               unsigned OpNo) {
    1196             : 
    1197          25 :   SDValue DataOp = N->getValue();
    1198          50 :   EVT DataVT = DataOp.getValueType();
    1199          50 :   SDValue Mask = N->getMask();
    1200          50 :   SDLoc dl(N);
    1201             : 
    1202          25 :   bool TruncateStore = false;
    1203          25 :   if (OpNo == 2) {
    1204             :     // Mask comes before the data operand. If the data operand is legal, we just
    1205             :     // promote the mask.
    1206             :     // When the data operand has illegal type, we should legalize the data
    1207             :     // operand first. The mask will be promoted/splitted/widened according to
    1208             :     // the data operand type.
    1209          35 :     if (TLI.isTypeLegal(DataVT))
    1210          14 :       Mask = PromoteTargetBoolean(Mask, DataVT);
    1211             :     else {
    1212          14 :       if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
    1213           3 :         return PromoteIntOp_MSTORE(N, 3);
    1214             : 
    1215           8 :       else if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector)
    1216           4 :         return WidenVecOp_MSTORE(N, 3);
    1217             : 
    1218             :       else {
    1219             :         assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
    1220           0 :         return SplitVecOp_MSTORE(N, 3);
    1221             :       }
    1222             :     }
    1223             :   } else { // Data operand
    1224             :     assert(OpNo == 3 && "Unexpected operand for promotion");
    1225           4 :     DataOp = GetPromotedInteger(DataOp);
    1226           8 :     Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());
    1227           4 :     TruncateStore = true;
    1228             :   }
    1229             : 
    1230          72 :   return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
    1231             :                             N->getMemoryVT(), N->getMemOperand(),
    1232          54 :                             TruncateStore, N->isCompressingStore());
    1233             : }
    1234             : 
    1235          45 : SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
    1236             :                                              unsigned OpNo) {
    1237             :   assert(OpNo == 2 && "Only know how to promote the mask!");
    1238          90 :   EVT DataVT = N->getValueType(0);
    1239          90 :   SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
    1240         180 :   SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
    1241          90 :   NewOps[OpNo] = Mask;
    1242         135 :   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1243             : }
    1244             : 
    1245          42 : SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
    1246             :                                                unsigned OpNo) {
    1247             : 
    1248         168 :   SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
    1249          42 :   if (OpNo == 2) {
    1250             :     // The Mask
    1251          40 :     EVT DataVT = N->getValueType(0);
    1252          60 :     NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
    1253             :   } else
    1254          66 :     NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
    1255             : 
    1256         126 :   SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1257             :   // updated in place.
    1258          42 :   if (Res.getNode() == N)
    1259          40 :     return Res;
    1260             : 
    1261           6 :   ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
    1262           6 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    1263           2 :   return SDValue();
    1264             : }
    1265             : 
    1266          22 : SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
    1267             :                                                 unsigned OpNo) {
    1268          88 :   SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
    1269          22 :   if (OpNo == 2) {
    1270             :     // The Mask
    1271          24 :     EVT DataVT = N->getValue().getValueType();
    1272          24 :     NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
    1273             :   } else
    1274          42 :     NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
    1275          66 :   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
    1276             : }
    1277             : 
    1278        2819 : SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
    1279        5638 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1280       14095 :   return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
    1281             : }
    1282             : 
    1283        1559 : SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
    1284        3118 :   return SDValue(DAG.UpdateNodeOperands(N,
    1285        4677 :                                 ZExtPromotedInteger(N->getOperand(0))), 0);
    1286             : }
    1287             : 
    1288       66897 : SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
    1289      133794 :   SDLoc dl(N);
    1290      133794 :   SDValue Op = GetPromotedInteger(N->getOperand(0));
    1291      200691 :   Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
    1292       66897 :   return DAG.getZeroExtendInReg(Op, dl,
    1293      267588 :                                 N->getOperand(0).getValueType().getScalarType());
    1294             : }
    1295             : 
    1296           5 : SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
    1297             :   assert(OpNo == 2 && "Don't know how to promote this operand!");
    1298             : 
    1299          10 :   SDValue LHS = N->getOperand(0);
    1300          10 :   SDValue RHS = N->getOperand(1);
    1301          10 :   SDValue Carry = N->getOperand(2);
    1302          10 :   SDLoc DL(N);
    1303             : 
    1304          10 :   auto VT = getSetCCResultType(LHS.getValueType());
    1305           5 :   TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT);
    1306           5 :   switch (BoolType) {
    1307           0 :   case TargetLoweringBase::UndefinedBooleanContent:
    1308           0 :     Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
    1309           0 :     break;
    1310           5 :   case TargetLoweringBase::ZeroOrOneBooleanContent:
    1311           5 :     Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
    1312           5 :     break;
    1313           0 :   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
    1314           0 :     Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
    1315           0 :     break;
    1316             :   }
    1317             : 
    1318          10 :   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
    1319             : }
    1320             : 
    1321             : //===----------------------------------------------------------------------===//
    1322             : //  Integer Result Expansion
    1323             : //===----------------------------------------------------------------------===//
    1324             : 
    1325             : /// ExpandIntegerResult - This method is called when the specified result of the
    1326             : /// specified node is found to need expansion.  At this point, the node may also
    1327             : /// have invalid operands or may have other results that need promotion, we just
    1328             : /// know that (at least) one result needs expansion.
    1329      327034 : void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
    1330             :   DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
    1331             :   SDValue Lo, Hi;
    1332      327034 :   Lo = Hi = SDValue();
    1333             : 
    1334             :   // See if the target wants to custom expand this node.
    1335      654068 :   if (CustomLowerNode(N, N->getValueType(ResNo), true))
    1336         660 :     return;
    1337             : 
    1338      652748 :   switch (N->getOpcode()) {
    1339           0 :   default:
    1340             : #ifndef NDEBUG
    1341             :     dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
    1342             :     N->dump(&DAG); dbgs() << "\n";
    1343             : #endif
    1344           0 :     llvm_unreachable("Do not know how to expand the result of this operator!");
    1345             : 
    1346          40 :   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
    1347         431 :   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
    1348         620 :   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
    1349        1614 :   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
    1350             : 
    1351        9579 :   case ISD::BITCAST:            ExpandRes_BITCAST(N, Lo, Hi); break;
    1352       11241 :   case ISD::BUILD_PAIR:         ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
    1353        2732 :   case ISD::EXTRACT_ELEMENT:    ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
    1354        1858 :   case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
    1355          14 :   case ISD::VAARG:              ExpandRes_VAARG(N, Lo, Hi); break;
    1356             : 
    1357        3311 :   case ISD::ANY_EXTEND:  ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
    1358         317 :   case ISD::AssertSext:  ExpandIntRes_AssertSext(N, Lo, Hi); break;
    1359         315 :   case ISD::AssertZext:  ExpandIntRes_AssertZext(N, Lo, Hi); break;
    1360           1 :   case ISD::BITREVERSE:  ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
    1361          43 :   case ISD::BSWAP:       ExpandIntRes_BSWAP(N, Lo, Hi); break;
    1362       64739 :   case ISD::Constant:    ExpandIntRes_Constant(N, Lo, Hi); break;
    1363          56 :   case ISD::CTLZ_ZERO_UNDEF:
    1364          56 :   case ISD::CTLZ:        ExpandIntRes_CTLZ(N, Lo, Hi); break;
    1365          11 :   case ISD::CTPOP:       ExpandIntRes_CTPOP(N, Lo, Hi); break;
    1366           8 :   case ISD::CTTZ_ZERO_UNDEF:
    1367           8 :   case ISD::CTTZ:        ExpandIntRes_CTTZ(N, Lo, Hi); break;
    1368           1 :   case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
    1369          89 :   case ISD::FP_TO_SINT:  ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
    1370          81 :   case ISD::FP_TO_UINT:  ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
    1371      136302 :   case ISD::LOAD:        ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
    1372         934 :   case ISD::MUL:         ExpandIntRes_MUL(N, Lo, Hi); break;
    1373           2 :   case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
    1374          66 :   case ISD::SDIV:        ExpandIntRes_SDIV(N, Lo, Hi); break;
    1375        1590 :   case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
    1376         156 :   case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
    1377          73 :   case ISD::SREM:        ExpandIntRes_SREM(N, Lo, Hi); break;
    1378       12305 :   case ISD::TRUNCATE:    ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
    1379          59 :   case ISD::UDIV:        ExpandIntRes_UDIV(N, Lo, Hi); break;
    1380          76 :   case ISD::UREM:        ExpandIntRes_UREM(N, Lo, Hi); break;
    1381        5298 :   case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
    1382          11 :   case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
    1383             : 
    1384          41 :   case ISD::ATOMIC_LOAD_ADD:
    1385             :   case ISD::ATOMIC_LOAD_SUB:
    1386             :   case ISD::ATOMIC_LOAD_AND:
    1387             :   case ISD::ATOMIC_LOAD_OR:
    1388             :   case ISD::ATOMIC_LOAD_XOR:
    1389             :   case ISD::ATOMIC_LOAD_NAND:
    1390             :   case ISD::ATOMIC_LOAD_MIN:
    1391             :   case ISD::ATOMIC_LOAD_MAX:
    1392             :   case ISD::ATOMIC_LOAD_UMIN:
    1393             :   case ISD::ATOMIC_LOAD_UMAX:
    1394             :   case ISD::ATOMIC_SWAP:
    1395             :   case ISD::ATOMIC_CMP_SWAP: {
    1396          41 :     std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
    1397          41 :     SplitInteger(Tmp.first, Lo, Hi);
    1398          41 :     ReplaceValueWith(SDValue(N, 1), Tmp.second);
    1399             :     break;
    1400             :   }
    1401          34 :   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
    1402          34 :     AtomicSDNode *AN = cast<AtomicSDNode>(N);
    1403         102 :     SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
    1404          34 :     SDValue Tmp = DAG.getAtomicCmpSwap(
    1405          68 :         ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
    1406         170 :         N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
    1407         102 :         AN->getMemOperand());
    1408             : 
    1409             :     // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
    1410             :     // success simply by comparing the loaded value against the ingoing
    1411             :     // comparison.
    1412         102 :     SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
    1413         136 :                                    N->getOperand(2), ISD::SETEQ);
    1414             : 
    1415          34 :     SplitInteger(Tmp, Lo, Hi);
    1416          34 :     ReplaceValueWith(SDValue(N, 1), Success);
    1417         102 :     ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
    1418             :     break;
    1419             :   }
    1420             : 
    1421       14866 :   case ISD::AND:
    1422             :   case ISD::OR:
    1423       14866 :   case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
    1424             : 
    1425           9 :   case ISD::UMAX:
    1426             :   case ISD::SMAX:
    1427             :   case ISD::UMIN:
    1428           9 :   case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
    1429             : 
    1430       65479 :   case ISD::ADD:
    1431       65479 :   case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
    1432             : 
    1433          23 :   case ISD::ADDC:
    1434          23 :   case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
    1435             : 
    1436          27 :   case ISD::ADDE:
    1437          27 :   case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
    1438             : 
    1439         742 :   case ISD::ADDCARRY:
    1440         742 :   case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
    1441             : 
    1442       58764 :   case ISD::SHL:
    1443             :   case ISD::SRA:
    1444       58764 :   case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
    1445             : 
    1446          20 :   case ISD::SADDO:
    1447          20 :   case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
    1448         536 :   case ISD::UADDO:
    1449         536 :   case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
    1450          11 :   case ISD::UMULO:
    1451          11 :   case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
    1452             :   }
    1453             : 
    1454             :   // If Lo/Hi is null, the sub-method took care of registering results etc.
    1455      326374 :   if (Lo.getNode())
    1456      326363 :     SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
    1457             : }
    1458             : 
    1459             : /// Lower an atomic node to the appropriate builtin call.
    1460          41 : std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
    1461          82 :   unsigned Opc = Node->getOpcode();
    1462          41 :   MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
    1463          41 :   RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
    1464             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
    1465             : 
    1466          41 :   return ExpandChainLibCall(LC, Node, false);
    1467             : }
    1468             : 
    1469             : /// N is a shift by a value that needs to be expanded,
    1470             : /// and the shift amount is a constant 'Amt'.  Expand the operation.
    1471       58241 : void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
    1472             :                                              SDValue &Lo, SDValue &Hi) {
    1473       58309 :   SDLoc DL(N);
    1474             :   // Expand the incoming operand to be shifted, so that we have its parts
    1475       58241 :   SDValue InL, InH;
    1476      116482 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    1477             : 
    1478             :   // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
    1479             :   // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
    1480       58241 :   if (!Amt) {
    1481           0 :     Lo = InL;
    1482           0 :     Hi = InH;
    1483       58173 :     return;
    1484             :   }
    1485             : 
    1486      116482 :   EVT NVT = InL.getValueType();
    1487      116482 :   unsigned VTBits = N->getValueType(0).getSizeInBits();
    1488       58241 :   unsigned NVTBits = NVT.getSizeInBits();
    1489      174723 :   EVT ShTy = N->getOperand(1).getValueType();
    1490             : 
    1491      116482 :   if (N->getOpcode() == ISD::SHL) {
    1492       12592 :     if (Amt.ugt(VTBits)) {
    1493           2 :       Lo = Hi = DAG.getConstant(0, DL, NVT);
    1494       12590 :     } else if (Amt.ugt(NVTBits)) {
    1495        2602 :       Lo = DAG.getConstant(0, DL, NVT);
    1496        5204 :       Hi = DAG.getNode(ISD::SHL, DL,
    1497       13010 :                        NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
    1498        9988 :     } else if (Amt == NVTBits) {
    1499        9295 :       Lo = DAG.getConstant(0, DL, NVT);
    1500        9295 :       Hi = InL;
    1501             :     } else {
    1502        1386 :       Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
    1503        1386 :       Hi = DAG.getNode(ISD::OR, DL, NVT,
    1504         693 :                        DAG.getNode(ISD::SHL, DL, NVT, InH,
    1505             :                                    DAG.getConstant(Amt, DL, ShTy)),
    1506         693 :                        DAG.getNode(ISD::SRL, DL, NVT, InL,
    1507        6930 :                                    DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
    1508             :     }
    1509             :     return;
    1510             :   }
    1511             : 
    1512       45649 :   if (N->getOpcode() == ISD::SRL) {
    1513       45581 :     if (Amt.ugt(VTBits)) {
    1514           2 :       Lo = Hi = DAG.getConstant(0, DL, NVT);
    1515       45579 :     } else if (Amt.ugt(NVTBits)) {
    1516         112 :       Lo = DAG.getNode(ISD::SRL, DL,
    1517         280 :                        NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
    1518          56 :       Hi = DAG.getConstant(0, DL, NVT);
    1519       45523 :     } else if (Amt == NVTBits) {
    1520       28164 :       Lo = InH;
    1521       28164 :       Hi = DAG.getConstant(0, DL, NVT);
    1522             :     } else {
    1523       34718 :       Lo = DAG.getNode(ISD::OR, DL, NVT,
    1524       17359 :                        DAG.getNode(ISD::SRL, DL, NVT, InL,
    1525             :                                    DAG.getConstant(Amt, DL, ShTy)),
    1526       17359 :                        DAG.getNode(ISD::SHL, DL, NVT, InH,
    1527      173590 :                                    DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
    1528       34718 :       Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
    1529             :     }
    1530             :     return;
    1531             :   }
    1532             : 
    1533             :   assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
    1534          68 :   if (Amt.ugt(VTBits)) {
    1535           4 :     Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1536           4 :                           DAG.getConstant(NVTBits - 1, DL, ShTy));
    1537          66 :   } else if (Amt.ugt(NVTBits)) {
    1538         114 :     Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1539         285 :                      DAG.getConstant(Amt - NVTBits, DL, ShTy));
    1540         114 :     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1541         114 :                      DAG.getConstant(NVTBits - 1, DL, ShTy));
    1542           9 :   } else if (Amt == NVTBits) {
    1543           3 :     Lo = InH;
    1544           6 :     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
    1545           6 :                      DAG.getConstant(NVTBits - 1, DL, ShTy));
    1546             :   } else {
    1547          12 :     Lo = DAG.getNode(ISD::OR, DL, NVT,
    1548           6 :                      DAG.getNode(ISD::SRL, DL, NVT, InL,
    1549             :                                  DAG.getConstant(Amt, DL, ShTy)),
    1550           6 :                      DAG.getNode(ISD::SHL, DL, NVT, InH,
    1551          60 :                                  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
    1552          12 :     Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
    1553             :   }
    1554             : }
    1555             : 
    1556             : /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
    1557             : /// this shift based on knowledge of the high bit of the shift amount.  If we
    1558             : /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
    1559             : /// shift amount.
    1560         523 : bool DAGTypeLegalizer::
    1561             : ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
    1562        1046 :   SDValue Amt = N->getOperand(1);
    1563        1569 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    1564        1046 :   EVT ShTy = Amt.getValueType();
    1565         523 :   unsigned ShBits = ShTy.getScalarSizeInBits();
    1566         523 :   unsigned NVTBits = NVT.getScalarSizeInBits();
    1567             :   assert(isPowerOf2_32(NVTBits) &&
    1568             :          "Expanded integer type size not a power of two!");
    1569        1046 :   SDLoc dl(N);
    1570             : 
    1571        1046 :   APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
    1572        1046 :   KnownBits Known;
    1573        1046 :   DAG.computeKnownBits(N->getOperand(1), Known);
    1574             : 
    1575             :   // If we don't know anything about the high bits, exit.
    1576        3138 :   if (((Known.Zero|Known.One) & HighBitMask) == 0)
    1577             :     return false;
    1578             : 
    1579             :   // Get the incoming operand to be shifted.
    1580          26 :   SDValue InL, InH;
    1581          52 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    1582             : 
    1583             :   // If we know that any of the high bits of the shift amount are one, then we
    1584             :   // can do this as a couple of simple shifts.
    1585          26 :   if (Known.One.intersects(HighBitMask)) {
    1586             :     // Mask out the high bit, which we know is set.
    1587           4 :     Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
    1588          10 :                       DAG.getConstant(~HighBitMask, dl, ShTy));
    1589             : 
    1590           4 :     switch (N->getOpcode()) {
    1591           0 :     default: llvm_unreachable("Unknown shift");
    1592           0 :     case ISD::SHL:
    1593           0 :       Lo = DAG.getConstant(0, dl, NVT);              // Low part is zero.
    1594           0 :       Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
    1595           0 :       return true;
    1596           2 :     case ISD::SRL:
    1597           2 :       Hi = DAG.getConstant(0, dl, NVT);              // Hi part is zero.
    1598           4 :       Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
    1599           2 :       return true;
    1600           0 :     case ISD::SRA:
    1601           0 :       Hi = DAG.getNode(ISD::SRA, dl, NVT, InH,       // Sign extend high part.
    1602           0 :                        DAG.getConstant(NVTBits - 1, dl, ShTy));
    1603           0 :       Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
    1604           0 :       return true;
    1605             :     }
    1606             :   }
    1607             : 
    1608             :   // If we know that all of the high bits of the shift amount are zero, then we
    1609             :   // can do this as a couple of simple shifts.
    1610          24 :   if (HighBitMask.isSubsetOf(Known.Zero)) {
    1611             :     // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
    1612             :     // shift if x is zero.  We can use XOR here because x is known to be smaller
    1613             :     // than 32.
    1614          11 :     SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
    1615          11 :                                DAG.getConstant(NVTBits - 1, dl, ShTy));
    1616             : 
    1617             :     unsigned Op1, Op2;
    1618          22 :     switch (N->getOpcode()) {
    1619           0 :     default: llvm_unreachable("Unknown shift");
    1620             :     case ISD::SHL:  Op1 = ISD::SHL; Op2 = ISD::SRL; break;
    1621           6 :     case ISD::SRL:
    1622           6 :     case ISD::SRA:  Op1 = ISD::SRL; Op2 = ISD::SHL; break;
    1623             :     }
    1624             : 
    1625             :     // When shifting right the arithmetic for Lo and Hi is swapped.
    1626          11 :     if (N->getOpcode() != ISD::SHL)
    1627             :       std::swap(InL, InH);
    1628             : 
    1629             :     // Use a little trick to get the bits that move from Lo to Hi. First
    1630             :     // shift by one bit.
    1631          22 :     SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
    1632             :     // Then compute the remaining shift with amount-1.
    1633          22 :     SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
    1634             : 
    1635          33 :     Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
    1636          33 :     Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
    1637             : 
    1638          11 :     if (N->getOpcode() != ISD::SHL)
    1639             :       std::swap(Hi, Lo);
    1640             :     return true;
    1641             :   }
    1642             : 
    1643             :   return false;
    1644             : }
    1645             : 
    1646             : /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
    1647             : /// of any size.
    1648          64 : bool DAGTypeLegalizer::
    1649             : ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
    1650         128 :   SDValue Amt = N->getOperand(1);
    1651         192 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    1652         128 :   EVT ShTy = Amt.getValueType();
    1653          64 :   unsigned NVTBits = NVT.getSizeInBits();
    1654             :   assert(isPowerOf2_32(NVTBits) &&
    1655             :          "Expanded integer type size not a power of two!");
    1656         128 :   SDLoc dl(N);
    1657             : 
    1658             :   // Get the incoming operand to be shifted.
    1659          64 :   SDValue InL, InH;
    1660         128 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    1661             : 
    1662          64 :   SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
    1663         128 :   SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
    1664         128 :   SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
    1665          64 :   SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
    1666          64 :                                  Amt, NVBitsNode, ISD::SETULT);
    1667          64 :   SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
    1668             :                                 Amt, DAG.getConstant(0, dl, ShTy),
    1669          64 :                                 ISD::SETEQ);
    1670             : 
    1671          64 :   SDValue LoS, HiS, LoL, HiL;
    1672         128 :   switch (N->getOpcode()) {
    1673           0 :   default: llvm_unreachable("Unknown shift");
    1674          26 :   case ISD::SHL:
    1675             :     // Short: ShAmt < NVTBits
    1676          52 :     LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
    1677          52 :     HiS = DAG.getNode(ISD::OR, dl, NVT,
    1678          26 :                       DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
    1679         104 :                       DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
    1680             : 
    1681             :     // Long: ShAmt >= NVTBits
    1682          26 :     LoL = DAG.getConstant(0, dl, NVT);                    // Lo part is zero.
    1683          52 :     HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
    1684             : 
    1685          26 :     Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
    1686          52 :     Hi = DAG.getSelect(dl, NVT, isZero, InH,
    1687          26 :                        DAG.getSelect(dl, NVT, isShort, HiS, HiL));
    1688          26 :     return true;
    1689          21 :   case ISD::SRL:
    1690             :     // Short: ShAmt < NVTBits
    1691          42 :     HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
    1692          42 :     LoS = DAG.getNode(ISD::OR, dl, NVT,
    1693          21 :                       DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
    1694             :     // FIXME: If Amt is zero, the following shift generates an undefined result
    1695             :     // on some architectures.
    1696          84 :                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
    1697             : 
    1698             :     // Long: ShAmt >= NVTBits
    1699          21 :     HiL = DAG.getConstant(0, dl, NVT);                    // Hi part is zero.
    1700          42 :     LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
    1701             : 
    1702          42 :     Lo = DAG.getSelect(dl, NVT, isZero, InL,
    1703          21 :                        DAG.getSelect(dl, NVT, isShort, LoS, LoL));
    1704          21 :     Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
    1705          21 :     return true;
    1706          17 :   case ISD::SRA:
    1707             :     // Short: ShAmt < NVTBits
    1708          34 :     HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
    1709          34 :     LoS = DAG.getNode(ISD::OR, dl, NVT,
    1710          17 :                       DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
    1711          68 :                       DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
    1712             : 
    1713             :     // Long: ShAmt >= NVTBits
    1714          34 :     HiL = DAG.getNode(ISD::SRA, dl, NVT, InH,             // Sign of Hi part.
    1715          34 :                       DAG.getConstant(NVTBits - 1, dl, ShTy));
    1716          34 :     LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
    1717             : 
    1718          34 :     Lo = DAG.getSelect(dl, NVT, isZero, InL,
    1719          17 :                        DAG.getSelect(dl, NVT, isShort, LoS, LoL));
    1720          17 :     Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
    1721          17 :     return true;
    1722             :   }
    1723             : }
    1724             : 
    1725           9 : static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
    1726             : 
    1727           9 :   switch (Op) {
    1728           0 :     default: llvm_unreachable("invalid min/max opcode");
    1729           2 :     case ISD::SMAX:
    1730           2 :       return std::make_pair(ISD::SETGT, ISD::UMAX);
    1731           3 :     case ISD::UMAX:
    1732           3 :       return std::make_pair(ISD::SETUGT, ISD::UMAX);
    1733           2 :     case ISD::SMIN:
    1734           2 :       return std::make_pair(ISD::SETLT, ISD::UMIN);
    1735           2 :     case ISD::UMIN:
    1736           2 :       return std::make_pair(ISD::SETULT, ISD::UMIN);
    1737             :   }
    1738             : }
    1739             : 
    1740           9 : void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
    1741             :                                            SDValue &Lo, SDValue &Hi) {
    1742          18 :   SDLoc DL(N);
    1743             :   ISD::NodeType LoOpc;
    1744             :   ISD::CondCode CondC;
    1745          36 :   std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
    1746             : 
    1747             :   // Expand the subcomponents.
    1748           9 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1749          18 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1750          18 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1751             : 
    1752             :   // Value types
    1753          18 :   EVT NVT = LHSL.getValueType();
    1754           9 :   EVT CCT = getSetCCResultType(NVT);
    1755             : 
    1756             :   // Hi part is always the same op
    1757          36 :   Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
    1758             : 
    1759             :   // We need to know whether to select Lo part that corresponds to 'winning'
    1760             :   // Hi part or if Hi parts are equal.
    1761           9 :   SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
    1762           9 :   SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
    1763             : 
    1764             :   // Lo part corresponding to the 'winning' Hi part
    1765           9 :   SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
    1766             : 
    1767             :   // Recursed Lo part if Hi parts are equal, this uses unsigned version
    1768          27 :   SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
    1769             : 
    1770           9 :   Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
    1771           9 : }
    1772             : 
    1773       65479 : void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
    1774             :                                            SDValue &Lo, SDValue &Hi) {
    1775       65546 :   SDLoc dl(N);
    1776             :   // Expand the subcomponents.
    1777       65479 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1778      130958 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1779      130958 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1780             : 
    1781      130958 :   EVT NVT = LHSL.getValueType();
    1782       65479 :   SDValue LoOps[2] = { LHSL, RHSL };
    1783       65479 :   SDValue HiOps[3] = { LHSH, RHSH };
    1784             : 
    1785      196437 :   bool HasOpCarry = TLI.isOperationLegalOrCustom(
    1786       65479 :       N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
    1787      130958 :       TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
    1788       65479 :   if (HasOpCarry) {
    1789       63878 :     SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
    1790       63878 :     if (N->getOpcode() == ISD::ADD) {
    1791      127638 :       Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
    1792      127638 :       HiOps[2] = Lo.getValue(1);
    1793      127638 :       Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
    1794             :     } else {
    1795         118 :       Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
    1796         118 :       HiOps[2] = Lo.getValue(1);
    1797         118 :       Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
    1798             :     }
    1799             :     return;
    1800             :   }
    1801             : 
    1802             :   // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
    1803             :   // them.  TODO: Teach operation legalization how to expand unsupported
    1804             :   // ADDC/ADDE/SUBC/SUBE.  The problem is that these operations generate
    1805             :   // a carry of type MVT::Glue, but there doesn't seem to be any way to
    1806             :   // generate a value of this type in the expanded code sequence.
    1807             :   bool hasCarry =
    1808        3202 :     TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
    1809             :                                    ISD::ADDC : ISD::SUBC,
    1810        3202 :                                  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
    1811             : 
    1812        1601 :   if (hasCarry) {
    1813        1314 :     SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
    1814         657 :     if (N->getOpcode() == ISD::ADD) {
    1815        1120 :       Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
    1816        1120 :       HiOps[2] = Lo.getValue(1);
    1817        1120 :       Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
    1818             :     } else {
    1819         194 :       Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
    1820         194 :       HiOps[2] = Lo.getValue(1);
    1821         194 :       Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
    1822             :     }
    1823             :     return;
    1824             :   }
    1825             : 
    1826             :   bool hasOVF =
    1827        1888 :     TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
    1828             :                                    ISD::UADDO : ISD::USUBO,
    1829        1888 :                                  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
    1830         944 :   TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
    1831             : 
    1832         944 :   if (hasOVF) {
    1833         686 :     EVT OvfVT = getSetCCResultType(NVT);
    1834         686 :     SDVTList VTList = DAG.getVTList(NVT, OvfVT);
    1835             :     int RevOpc;
    1836         686 :     if (N->getOpcode() == ISD::ADD) {
    1837          59 :       RevOpc = ISD::SUB;
    1838         118 :       Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
    1839         177 :       Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
    1840             :     } else {
    1841         627 :       RevOpc = ISD::ADD;
    1842        1254 :       Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
    1843        1881 :       Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
    1844             :     }
    1845        1372 :     SDValue OVF = Lo.getValue(1);
    1846             : 
    1847         686 :     switch (BoolType) {
    1848           0 :     case TargetLoweringBase::UndefinedBooleanContent:
    1849           0 :       OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
    1850             :       LLVM_FALLTHROUGH;
    1851           0 :     case TargetLoweringBase::ZeroOrOneBooleanContent:
    1852           0 :       OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
    1853           0 :       Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
    1854           0 :       break;
    1855         686 :     case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
    1856         686 :       OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
    1857        1372 :       Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
    1858             :     }
    1859             :     return;
    1860             :   }
    1861             : 
    1862         258 :   if (N->getOpcode() == ISD::ADD) {
    1863         579 :     Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
    1864         579 :     Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
    1865         193 :     SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
    1866         193 :                                 ISD::SETULT);
    1867             : 
    1868         193 :     if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
    1869         191 :       SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
    1870         382 :       Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
    1871             :       return;
    1872             :     }
    1873             : 
    1874           2 :     SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
    1875           2 :                                    DAG.getConstant(1, dl, NVT),
    1876           4 :                                    DAG.getConstant(0, dl, NVT));
    1877           2 :     SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
    1878           2 :                                 ISD::SETULT);
    1879           2 :     SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
    1880           2 :                                    DAG.getConstant(1, dl, NVT), Carry1);
    1881           4 :     Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
    1882             :   } else {
    1883         195 :     Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
    1884         195 :     Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
    1885             :     SDValue Cmp =
    1886          65 :       DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
    1887         130 :                    LoOps[0], LoOps[1], ISD::SETULT);
    1888             : 
    1889          65 :     SDValue Borrow;
    1890          65 :     if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
    1891          64 :       Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
    1892             :     else
    1893           3 :       Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
    1894           2 :                              DAG.getConstant(0, dl, NVT));
    1895             : 
    1896         130 :     Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
    1897             :   }
    1898             : }
    1899             : 
    1900          23 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
    1901             :                                             SDValue &Lo, SDValue &Hi) {
    1902             :   // Expand the subcomponents.
    1903          23 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1904          46 :   SDLoc dl(N);
    1905          46 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1906          46 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1907          69 :   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
    1908          23 :   SDValue LoOps[2] = { LHSL, RHSL };
    1909          23 :   SDValue HiOps[3] = { LHSH, RHSH };
    1910             : 
    1911          23 :   if (N->getOpcode() == ISD::ADDC) {
    1912          42 :     Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
    1913          42 :     HiOps[2] = Lo.getValue(1);
    1914          42 :     Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
    1915             :   } else {
    1916           4 :     Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
    1917           4 :     HiOps[2] = Lo.getValue(1);
    1918           4 :     Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
    1919             :   }
    1920             : 
    1921             :   // Legalized the flag result - switch anything that used the old flag to
    1922             :   // use the new one.
    1923          69 :   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
    1924          23 : }
    1925             : 
    1926          27 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
    1927             :                                             SDValue &Lo, SDValue &Hi) {
    1928             :   // Expand the subcomponents.
    1929          27 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1930          54 :   SDLoc dl(N);
    1931          54 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1932          54 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1933          81 :   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
    1934          54 :   SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
    1935          27 :   SDValue HiOps[3] = { LHSH, RHSH };
    1936             : 
    1937          81 :   Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
    1938          54 :   HiOps[2] = Lo.getValue(1);
    1939          81 :   Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
    1940             : 
    1941             :   // Legalized the flag result - switch anything that used the old flag to
    1942             :   // use the new one.
    1943          81 :   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
    1944          27 : }
    1945             : 
    1946         536 : void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
    1947             :                                              SDValue &Lo, SDValue &Hi) {
    1948        1072 :   SDValue LHS = N->getOperand(0);
    1949        1072 :   SDValue RHS = N->getOperand(1);
    1950        1072 :   SDLoc dl(N);
    1951             : 
    1952         536 :   SDValue Ovf;
    1953             : 
    1954        1608 :   bool HasOpCarry = TLI.isOperationLegalOrCustom(
    1955         536 :       N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
    1956        1072 :       TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
    1957             : 
    1958         536 :   if (HasOpCarry) {
    1959             :     // Expand the subcomponents.
    1960         530 :     SDValue LHSL, LHSH, RHSL, RHSH;
    1961         530 :     GetExpandedInteger(LHS, LHSL, LHSH);
    1962         530 :     GetExpandedInteger(RHS, RHSL, RHSH);
    1963        1590 :     SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
    1964         530 :     SDValue LoOps[2] = { LHSL, RHSL };
    1965         530 :     SDValue HiOps[3] = { LHSH, RHSH };
    1966             : 
    1967        1060 :     unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
    1968        1060 :     Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
    1969        1060 :     HiOps[2] = Lo.getValue(1);
    1970        1060 :     Hi = DAG.getNode(Opc, dl, VTList, HiOps);
    1971             : 
    1972        1060 :     Ovf = Hi.getValue(1);
    1973             :   } else {
    1974             :     // Expand the result by simply replacing it with the equivalent
    1975             :     // non-overflow-checking operation.
    1976           6 :     auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
    1977          18 :     SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
    1978           6 :     SplitInteger(Sum, Lo, Hi);
    1979             : 
    1980             :     // Calculate the overflow: addition overflows iff a + b < a, and subtraction
    1981             :     // overflows iff a - b > a.
    1982           6 :     auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
    1983          12 :     Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
    1984             :   }
    1985             : 
    1986             :   // Legalized the flag result - switch anything that used the old flag to
    1987             :   // use the new one.
    1988         536 :   ReplaceValueWith(SDValue(N, 1), Ovf);
    1989         536 : }
    1990             : 
    1991         742 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
    1992             :                                                 SDValue &Lo, SDValue &Hi) {
    1993             :   // Expand the subcomponents.
    1994         742 :   SDValue LHSL, LHSH, RHSL, RHSH;
    1995        1484 :   SDLoc dl(N);
    1996        1484 :   GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    1997        1484 :   GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
    1998        2226 :   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
    1999        1484 :   SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
    2000         742 :   SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
    2001             : 
    2002        2226 :   Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
    2003        1484 :   HiOps[2] = Lo.getValue(1);
    2004        2226 :   Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
    2005             : 
    2006             :   // Legalized the flag result - switch anything that used the old flag to
    2007             :   // use the new one.
    2008        2226 :   ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
    2009         742 : }
    2010             : 
    2011        3311 : void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
    2012             :                                                SDValue &Lo, SDValue &Hi) {
    2013        9933 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2014        6622 :   SDLoc dl(N);
    2015        6622 :   SDValue Op = N->getOperand(0);
    2016        6622 :   if (Op.getValueType().bitsLE(NVT)) {
    2017             :     // The low part is any extension of the input (which degenerates to a copy).
    2018        6594 :     Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
    2019        3297 :     Hi = DAG.getUNDEF(NVT);   // The high part is undefined.
    2020             :   } else {
    2021             :     // For example, extension of an i48 to an i64.  The operand type necessarily
    2022             :     // promotes to the result type, so will end up being expanded too.
    2023             :     assert(getTypeAction(Op.getValueType()) ==
    2024             :            TargetLowering::TypePromoteInteger &&
    2025             :            "Only know how to promote this result!");
    2026          14 :     SDValue Res = GetPromotedInteger(Op);
    2027             :     assert(Res.getValueType() == N->getValueType(0) &&
    2028             :            "Operand over promoted?");
    2029             :     // Split the promoted operand.  This will simplify when it is expanded.
    2030          14 :     SplitInteger(Res, Lo, Hi);
    2031             :   }
    2032        3311 : }
    2033             : 
    2034         317 : void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
    2035             :                                                SDValue &Lo, SDValue &Hi) {
    2036         634 :   SDLoc dl(N);
    2037         634 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2038         634 :   EVT NVT = Lo.getValueType();
    2039         951 :   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    2040         317 :   unsigned NVTBits = NVT.getSizeInBits();
    2041         317 :   unsigned EVTBits = EVT.getSizeInBits();
    2042             : 
    2043         317 :   if (NVTBits < EVTBits) {
    2044         508 :     Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
    2045         254 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2046         762 :                                                         EVTBits - NVTBits)));
    2047             :   } else {
    2048         126 :     Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
    2049             :     // The high part replicates the sign bit of Lo, make it explicit.
    2050         126 :     Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
    2051          63 :                      DAG.getConstant(NVTBits - 1, dl,
    2052         378 :                                      TLI.getPointerTy(DAG.getDataLayout())));
    2053             :   }
    2054         317 : }
    2055             : 
    2056         315 : void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
    2057             :                                                SDValue &Lo, SDValue &Hi) {
    2058         630 :   SDLoc dl(N);
    2059         630 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2060         630 :   EVT NVT = Lo.getValueType();
    2061         945 :   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    2062         315 :   unsigned NVTBits = NVT.getSizeInBits();
    2063         315 :   unsigned EVTBits = EVT.getSizeInBits();
    2064             : 
    2065         315 :   if (NVTBits < EVTBits) {
    2066         504 :     Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
    2067         252 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2068         756 :                                                         EVTBits - NVTBits)));
    2069             :   } else {
    2070         126 :     Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
    2071             :     // The high part must be zero, make it explicit.
    2072          63 :     Hi = DAG.getConstant(0, dl, NVT);
    2073             :   }
    2074         315 : }
    2075             : 
    2076           1 : void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
    2077             :                                                SDValue &Lo, SDValue &Hi) {
    2078           2 :   SDLoc dl(N);
    2079           2 :   GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
    2080           3 :   Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
    2081           3 :   Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
    2082           1 : }
    2083             : 
    2084          43 : void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
    2085             :                                           SDValue &Lo, SDValue &Hi) {
    2086          86 :   SDLoc dl(N);
    2087          86 :   GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
    2088         129 :   Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
    2089         129 :   Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
    2090          43 : }
    2091             : 
    2092       64739 : void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
    2093             :                                              SDValue &Lo, SDValue &Hi) {
    2094      194217 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2095       64739 :   unsigned NBitWidth = NVT.getSizeInBits();
    2096       64739 :   auto Constant = cast<ConstantSDNode>(N);
    2097       64739 :   const APInt &Cst = Constant->getAPIntValue();
    2098      129478 :   bool IsTarget = Constant->isTargetOpcode();
    2099       64739 :   bool IsOpaque = Constant->isOpaque();
    2100      129478 :   SDLoc dl(N);
    2101      129478 :   Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
    2102      194217 :   Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
    2103             :                        IsOpaque);
    2104       64739 : }
    2105             : 
    2106          56 : void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
    2107             :                                          SDValue &Lo, SDValue &Hi) {
    2108         112 :   SDLoc dl(N);
    2109             :   // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
    2110         112 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2111         112 :   EVT NVT = Lo.getValueType();
    2112             : 
    2113          56 :   SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
    2114          56 :                                    DAG.getConstant(0, dl, NVT), ISD::SETNE);
    2115             : 
    2116         168 :   SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
    2117         112 :   SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
    2118             : 
    2119         112 :   Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
    2120          56 :                      DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
    2121          56 :                                  DAG.getConstant(NVT.getSizeInBits(), dl,
    2122         168 :                                                  NVT)));
    2123          56 :   Hi = DAG.getConstant(0, dl, NVT);
    2124          56 : }
    2125             : 
    2126          11 : void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
    2127             :                                           SDValue &Lo, SDValue &Hi) {
    2128          22 :   SDLoc dl(N);
    2129             :   // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
    2130          22 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2131          22 :   EVT NVT = Lo.getValueType();
    2132          33 :   Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
    2133          44 :                    DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
    2134          11 :   Hi = DAG.getConstant(0, dl, NVT);
    2135          11 : }
    2136             : 
    2137           8 : void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
    2138             :                                          SDValue &Lo, SDValue &Hi) {
    2139          16 :   SDLoc dl(N);
    2140             :   // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
    2141          16 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2142          16 :   EVT NVT = Lo.getValueType();
    2143             : 
    2144           8 :   SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
    2145           8 :                                    DAG.getConstant(0, dl, NVT), ISD::SETNE);
    2146             : 
    2147          16 :   SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
    2148          24 :   SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
    2149             : 
    2150          16 :   Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
    2151           8 :                      DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
    2152           8 :                                  DAG.getConstant(NVT.getSizeInBits(), dl,
    2153          24 :                                                  NVT)));
    2154           8 :   Hi = DAG.getConstant(0, dl, NVT);
    2155           8 : }
    2156             : 
    2157           1 : void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
    2158             :                                                SDValue &Hi) {
    2159           2 :   SDLoc dl(N);
    2160           3 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2161           1 :   unsigned NBitWidth = NVT.getSizeInBits();
    2162             : 
    2163           2 :   EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
    2164           1 :   Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
    2165             :   // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
    2166           2 :   Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
    2167           2 :                    DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
    2168           1 : }
    2169             : 
    2170          89 : void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
    2171             :                                                SDValue &Hi) {
    2172         178 :   SDLoc dl(N);
    2173         178 :   EVT VT = N->getValueType(0);
    2174             : 
    2175         178 :   SDValue Op = N->getOperand(0);
    2176         267 :   if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
    2177           6 :     Op = GetPromotedFloat(Op);
    2178             : 
    2179         178 :   RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
    2180             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
    2181         178 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
    2182             :                Lo, Hi);
    2183          89 : }
    2184             : 
    2185          81 : void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
    2186             :                                                SDValue &Hi) {
    2187         162 :   SDLoc dl(N);
    2188         162 :   EVT VT = N->getValueType(0);
    2189             : 
    2190         162 :   SDValue Op = N->getOperand(0);
    2191         243 :   if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
    2192           4 :     Op = GetPromotedFloat(Op);
    2193             : 
    2194         162 :   RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
    2195             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
    2196         162 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
    2197             :                Lo, Hi);
    2198          81 : }
    2199             : 
    2200       68151 : void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
    2201             :                                          SDValue &Lo, SDValue &Hi) {
    2202       67496 :   if (ISD::isNormalLoad(N)) {
    2203       67496 :     ExpandRes_NormalLoad(N, Lo, Hi);
    2204       67496 :     return;
    2205             :   }
    2206             : 
    2207             :   assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
    2208             : 
    2209        1310 :   EVT VT = N->getValueType(0);
    2210        1310 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2211        1310 :   SDValue Ch  = N->getChain();
    2212         655 :   SDValue Ptr = N->getBasePtr();
    2213         655 :   ISD::LoadExtType ExtType = N->getExtensionType();
    2214        1310 :   unsigned Alignment = N->getAlignment();
    2215         655 :   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
    2216        1310 :   AAMDNodes AAInfo = N->getAAInfo();
    2217        1310 :   SDLoc dl(N);
    2218             : 
    2219             :   assert(NVT.isByteSized() && "Expanded type not byte sized!");
    2220             : 
    2221         655 :   if (N->getMemoryVT().bitsLE(NVT)) {
    2222         365 :     EVT MemVT = N->getMemoryVT();
    2223             : 
    2224        1095 :     Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
    2225         365 :                         Alignment, MMOFlags, AAInfo);
    2226             : 
    2227             :     // Remember the chain.
    2228         730 :     Ch = Lo.getValue(1);
    2229             : 
    2230         365 :     if (ExtType == ISD::SEXTLOAD) {
    2231             :       // The high part is obtained by SRA'ing all but one of the bits of the
    2232             :       // lo part.
    2233          71 :       unsigned LoSize = Lo.getValueSizeInBits();
    2234         142 :       Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
    2235          71 :                        DAG.getConstant(LoSize - 1, dl,
    2236         426 :                                        TLI.getPointerTy(DAG.getDataLayout())));
    2237         294 :     } else if (ExtType == ISD::ZEXTLOAD) {
    2238             :       // The high part is just a zero.
    2239         171 :       Hi = DAG.getConstant(0, dl, NVT);
    2240             :     } else {
    2241             :       assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
    2242             :       // The high part is undefined.
    2243         123 :       Hi = DAG.getUNDEF(NVT);
    2244             :     }
    2245         580 :   } else if (DAG.getDataLayout().isLittleEndian()) {
    2246             :     // Little-endian - low bits are at low addresses.
    2247         846 :     Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
    2248         282 :                      AAInfo);
    2249             : 
    2250             :     unsigned ExcessBits =
    2251         282 :       N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
    2252         282 :     EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
    2253             : 
    2254             :     // Increment the pointer to the other half.
    2255         282 :     unsigned IncrementSize = NVT.getSizeInBits()/8;
    2256         564 :     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
    2257         846 :                       DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
    2258         564 :     Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
    2259         282 :                         N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
    2260        1128 :                         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    2261             : 
    2262             :     // Build a factor node to remember that this load is independent of the
    2263             :     // other one.
    2264         564 :     Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    2265        1410 :                      Hi.getValue(1));
    2266             :   } else {
    2267             :     // Big-endian - high bits are at low addresses.  Favor aligned loads at
    2268             :     // the cost of some bit-fiddling.
    2269           8 :     EVT MemVT = N->getMemoryVT();
    2270           8 :     unsigned EBytes = MemVT.getStoreSize();
    2271           8 :     unsigned IncrementSize = NVT.getSizeInBits()/8;
    2272           8 :     unsigned ExcessBits = (EBytes - IncrementSize)*8;
    2273             : 
    2274             :     // Load both the high bits and maybe some of the low bits.
    2275          24 :     Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
    2276           8 :                         EVT::getIntegerVT(*DAG.getContext(),
    2277             :                                           MemVT.getSizeInBits() - ExcessBits),
    2278          16 :                         Alignment, MMOFlags, AAInfo);
    2279             : 
    2280             :     // Increment the pointer to the other half.
    2281          16 :     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
    2282          24 :                       DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
    2283             :     // Load the rest of the low bits.
    2284          16 :     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
    2285           8 :                         N->getPointerInfo().getWithOffset(IncrementSize),
    2286           8 :                         EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
    2287          40 :                         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    2288             : 
    2289             :     // Build a factor node to remember that this load is independent of the
    2290             :     // other one.
    2291          16 :     Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    2292          40 :                      Hi.getValue(1));
    2293             : 
    2294           8 :     if (ExcessBits < NVT.getSizeInBits()) {
    2295             :       // Transfer low bits from the bottom of Hi to the top of Lo.
    2296          14 :       Lo = DAG.getNode(
    2297             :           ISD::OR, dl, NVT, Lo,
    2298           7 :           DAG.getNode(ISD::SHL, dl, NVT, Hi,
    2299             :                       DAG.getConstant(ExcessBits, dl,
    2300          42 :                                       TLI.getPointerTy(DAG.getDataLayout()))));
    2301             :       // Move high bits to the right position in Hi.
    2302          14 :       Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
    2303             :                        Hi,
    2304           7 :                        DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
    2305          42 :                                        TLI.getPointerTy(DAG.getDataLayout())));
    2306             :     }
    2307             :   }
    2308             : 
    2309             :   // Legalize the chain result - switch anything that used the old chain to
    2310             :   // use the new one.
    2311         655 :   ReplaceValueWith(SDValue(N, 1), Ch);
    2312             : }
    2313             : 
    2314       14866 : void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
    2315             :                                             SDValue &Lo, SDValue &Hi) {
    2316       29732 :   SDLoc dl(N);
    2317       14866 :   SDValue LL, LH, RL, RH;
    2318       29732 :   GetExpandedInteger(N->getOperand(0), LL, LH);
    2319       29732 :   GetExpandedInteger(N->getOperand(1), RL, RH);
    2320       59464 :   Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
    2321       59464 :   Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
    2322       14866 : }
    2323             : 
    2324         934 : void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
    2325             :                                         SDValue &Lo, SDValue &Hi) {
    2326        1868 :   EVT VT = N->getValueType(0);
    2327        1868 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2328        1174 :   SDLoc dl(N);
    2329             : 
    2330         934 :   SDValue LL, LH, RL, RH;
    2331        1868 :   GetExpandedInteger(N->getOperand(0), LL, LH);
    2332        1868 :   GetExpandedInteger(N->getOperand(1), RL, RH);
    2333             : 
    2334         934 :   if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
    2335             :                     TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
    2336             :                     LL, LH, RL, RH))
    2337         694 :     return;
    2338             : 
    2339             :   // If nothing else, we can make a libcall.
    2340         340 :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2341         680 :   if (VT == MVT::i16)
    2342             :     LC = RTLIB::MUL_I16;
    2343         673 :   else if (VT == MVT::i32)
    2344             :     LC = RTLIB::MUL_I32;
    2345         621 :   else if (VT == MVT::i64)
    2346             :     LC = RTLIB::MUL_I64;
    2347         288 :   else if (VT == MVT::i128)
    2348             :     LC = RTLIB::MUL_I128;
    2349             : 
    2350         492 :   if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
    2351             :     // We'll expand the multiplication by brute force because we have no other
    2352             :     // options. This is a trivially-generalized version of the code from
    2353             :     // Hacker's Delight (itself derived from Knuth's Algorithm M from section
    2354             :     // 4.3.1).
    2355         100 :     unsigned Bits = NVT.getSizeInBits();
    2356         100 :     unsigned HalfBits = Bits >> 1;
    2357         200 :     SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
    2358         100 :                                    NVT);
    2359         200 :     SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
    2360         200 :     SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
    2361             : 
    2362         200 :     SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
    2363         200 :     SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
    2364             : 
    2365         200 :     EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
    2366         300 :     if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
    2367             :       // The type from TLI is too small to fit the shift amount we want.
    2368             :       // Override it with i32. The shift will have to be legalized.
    2369           2 :       ShiftAmtTy = MVT::i32;
    2370             :     }
    2371         100 :     SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
    2372         200 :     SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
    2373         200 :     SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
    2374         200 :     SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
    2375             : 
    2376         100 :     SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
    2377         200 :                             DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
    2378         200 :     SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
    2379         200 :     SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
    2380             : 
    2381         100 :     SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
    2382         200 :                             DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
    2383         200 :     SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
    2384             : 
    2385         100 :     SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
    2386         100 :                             DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
    2387         300 :                             DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
    2388         200 :     Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
    2389         300 :                      DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
    2390             : 
    2391         200 :     Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
    2392         100 :                      DAG.getNode(ISD::ADD, dl, NVT,
    2393         100 :                                  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
    2394         500 :                                  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
    2395             :     return;
    2396             :   }
    2397             : 
    2398         720 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2399         480 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
    2400             :                Lo, Hi);
    2401             : }
    2402             : 
    2403           2 : void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
    2404             :                                                      SDValue &Hi) {
    2405           4 :   SDLoc DL(N);
    2406           6 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2407           4 :   SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
    2408           6 :   SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
    2409           4 :   Lo = R.getValue(0);
    2410           4 :   Hi = R.getValue(1);
    2411           6 :   ReplaceValueWith(SDValue(N, 1), R.getValue(2));
    2412           2 : }
    2413             : 
    2414          20 : void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
    2415             :                                              SDValue &Lo, SDValue &Hi) {
    2416          40 :   SDValue LHS = Node->getOperand(0);
    2417          40 :   SDValue RHS = Node->getOperand(1);
    2418          40 :   SDLoc dl(Node);
    2419             : 
    2420             :   // Expand the result by simply replacing it with the equivalent
    2421             :   // non-overflow-checking operation.
    2422          40 :   SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
    2423             :                             ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
    2424          40 :                             LHS, RHS);
    2425          20 :   SplitInteger(Sum, Lo, Hi);
    2426             : 
    2427             :   // Compute the overflow.
    2428             :   //
    2429             :   //   LHSSign -> LHS >= 0
    2430             :   //   RHSSign -> RHS >= 0
    2431             :   //   SumSign -> Sum >= 0
    2432             :   //
    2433             :   //   Add:
    2434             :   //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
    2435             :   //   Sub:
    2436             :   //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
    2437             :   //
    2438          40 :   EVT OType = Node->getValueType(1);
    2439          40 :   SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
    2440             : 
    2441          20 :   SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
    2442          20 :   SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
    2443          20 :   SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
    2444          20 :                                     Node->getOpcode() == ISD::SADDO ?
    2445          20 :                                     ISD::SETEQ : ISD::SETNE);
    2446             : 
    2447          20 :   SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
    2448          20 :   SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
    2449             : 
    2450          40 :   SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
    2451             : 
    2452             :   // Use the calculated overflow everywhere.
    2453          20 :   ReplaceValueWith(SDValue(Node, 1), Cmp);
    2454          20 : }
    2455             : 
    2456          66 : void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
    2457             :                                          SDValue &Lo, SDValue &Hi) {
    2458         132 :   EVT VT = N->getValueType(0);
    2459         129 :   SDLoc dl(N);
    2460         198 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2461             : 
    2462         132 :   if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
    2463           6 :     SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2464           6 :     SplitInteger(Res.getValue(0), Lo, Hi);
    2465             :     return;
    2466             :   }
    2467             : 
    2468          63 :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2469         126 :   if (VT == MVT::i16)
    2470             :     LC = RTLIB::SDIV_I16;
    2471         126 :   else if (VT == MVT::i32)
    2472             :     LC = RTLIB::SDIV_I32;
    2473         124 :   else if (VT == MVT::i64)
    2474             :     LC = RTLIB::SDIV_I64;
    2475          17 :   else if (VT == MVT::i128)
    2476          17 :     LC = RTLIB::SDIV_I128;
    2477             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
    2478             : 
    2479         126 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
    2480             : }
    2481             : 
    2482       58764 : void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
    2483             :                                           SDValue &Lo, SDValue &Hi) {
    2484      117528 :   EVT VT = N->getValueType(0);
    2485       58828 :   SDLoc dl(N);
    2486             : 
    2487             :   // If we can emit an efficient shift operation, do so now.  Check to see if
    2488             :   // the RHS is a constant.
    2489      175769 :   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    2490      116941 :     return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
    2491             : 
    2492             :   // If we can determine that the high bit of the shift is zero or one, even if
    2493             :   // the low bits are variable, emit this shift in an optimized form.
    2494         523 :   if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
    2495             :     return;
    2496             : 
    2497             :   // If this target supports shift_PARTS, use it.  First, map to the _PARTS opc.
    2498             :   unsigned PartsOpc;
    2499        1020 :   if (N->getOpcode() == ISD::SHL) {
    2500             :     PartsOpc = ISD::SHL_PARTS;
    2501         278 :   } else if (N->getOpcode() == ISD::SRL) {
    2502             :     PartsOpc = ISD::SRL_PARTS;
    2503             :   } else {
    2504             :     assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
    2505          89 :     PartsOpc = ISD::SRA_PARTS;
    2506             :   }
    2507             : 
    2508             :   // Next check to see if the target supports this SHL_PARTS operation or if it
    2509             :   // will custom expand it.
    2510        1020 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2511        1020 :   TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
    2512        1020 :   if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
    2513             :       Action == TargetLowering::Custom) {
    2514             :     // Expand the subcomponents.
    2515         445 :     SDValue LHSL, LHSH;
    2516         890 :     GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
    2517         890 :     EVT VT = LHSL.getValueType();
    2518             : 
    2519             :     // If the shift amount operand is coming from a vector legalization it may
    2520             :     // have an illegal type.  Fix that first by casting the operand, otherwise
    2521             :     // the new SHL_PARTS operation would need further legalization.
    2522         890 :     SDValue ShiftOp = N->getOperand(1);
    2523         890 :     EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    2524             :     assert(ShiftTy.getScalarSizeInBits() >=
    2525             :            Log2_32_Ceil(VT.getScalarSizeInBits()) &&
    2526             :            "ShiftAmountTy is too small to cover the range of this type!");
    2527         890 :     if (ShiftOp.getValueType() != ShiftTy)
    2528          55 :       ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
    2529             : 
    2530         445 :     SDValue Ops[] = { LHSL, LHSH, ShiftOp };
    2531         890 :     Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
    2532         890 :     Hi = Lo.getValue(1);
    2533             :     return;
    2534             :   }
    2535             : 
    2536             :   // Otherwise, emit a libcall.
    2537          65 :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2538             :   bool isSigned;
    2539         130 :   if (N->getOpcode() == ISD::SHL) {
    2540          27 :     isSigned = false; /*sign irrelevant*/
    2541          54 :     if (VT == MVT::i16)
    2542             :       LC = RTLIB::SHL_I16;
    2543          54 :     else if (VT == MVT::i32)
    2544             :       LC = RTLIB::SHL_I32;
    2545          54 :     else if (VT == MVT::i64)
    2546             :       LC = RTLIB::SHL_I64;
    2547          27 :     else if (VT == MVT::i128)
    2548             :       LC = RTLIB::SHL_I128;
    2549          38 :   } else if (N->getOpcode() == ISD::SRL) {
    2550          21 :     isSigned = false;
    2551          42 :     if (VT == MVT::i16)
    2552             :       LC = RTLIB::SRL_I16;
    2553          42 :     else if (VT == MVT::i32)
    2554             :       LC = RTLIB::SRL_I32;
    2555          42 :     else if (VT == MVT::i64)
    2556             :       LC = RTLIB::SRL_I64;
    2557          21 :     else if (VT == MVT::i128)
    2558             :       LC = RTLIB::SRL_I128;
    2559             :   } else {
    2560             :     assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
    2561          17 :     isSigned = true;
    2562          34 :     if (VT == MVT::i16)
    2563             :       LC = RTLIB::SRA_I16;
    2564          34 :     else if (VT == MVT::i32)
    2565             :       LC = RTLIB::SRA_I32;
    2566          34 :     else if (VT == MVT::i64)
    2567             :       LC = RTLIB::SRA_I64;
    2568          17 :     else if (VT == MVT::i128)
    2569             :       LC = RTLIB::SRA_I128;
    2570             :   }
    2571             : 
    2572          94 :   if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
    2573           3 :     SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2574           2 :     SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
    2575             :     return;
    2576             :   }
    2577             : 
    2578          64 :   if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
    2579           0 :     llvm_unreachable("Unsupported shift!");
    2580             : }
    2581             : 
    2582        1590 : void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
    2583             :                                                 SDValue &Lo, SDValue &Hi) {
    2584        4770 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2585        3180 :   SDLoc dl(N);
    2586        3180 :   SDValue Op = N->getOperand(0);
    2587        3180 :   if (Op.getValueType().bitsLE(NVT)) {
    2588             :     // The low part is sign extension of the input (degenerates to a copy).
    2589        4767 :     Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
    2590             :     // The high part is obtained by SRA'ing all but one of the bits of low part.
    2591        1589 :     unsigned LoSize = NVT.getSizeInBits();
    2592        3178 :     Hi = DAG.getNode(
    2593             :         ISD::SRA, dl, NVT, Lo,
    2594        7945 :         DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
    2595             :   } else {
    2596             :     // For example, extension of an i48 to an i64.  The operand type necessarily
    2597             :     // promotes to the result type, so will end up being expanded too.
    2598             :     assert(getTypeAction(Op.getValueType()) ==
    2599             :            TargetLowering::TypePromoteInteger &&
    2600             :            "Only know how to promote this result!");
    2601           1 :     SDValue Res = GetPromotedInteger(Op);
    2602             :     assert(Res.getValueType() == N->getValueType(0) &&
    2603             :            "Operand over promoted?");
    2604             :     // Split the promoted operand.  This will simplify when it is expanded.
    2605           1 :     SplitInteger(Res, Lo, Hi);
    2606           1 :     unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
    2607           2 :     Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
    2608           1 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2609           3 :                                                         ExcessBits)));
    2610             :   }
    2611        1590 : }
    2612             : 
    2613         156 : void DAGTypeLegalizer::
    2614             : ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
    2615         312 :   SDLoc dl(N);
    2616         312 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    2617         468 :   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    2618             : 
    2619         312 :   if (EVT.bitsLE(Lo.getValueType())) {
    2620             :     // sext_inreg the low part if needed.
    2621         174 :     Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
    2622         261 :                      N->getOperand(1));
    2623             : 
    2624             :     // The high part gets the sign extension from the lo-part.  This handles
    2625             :     // things like sextinreg V:i64 from i8.
    2626         174 :     Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
    2627          87 :                      DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
    2628         522 :                                      TLI.getPointerTy(DAG.getDataLayout())));
    2629             :   } else {
    2630             :     // For example, extension of an i48 to an i64.  Leave the low part alone,
    2631             :     // sext_inreg the high part.
    2632          69 :     unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
    2633         138 :     Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
    2634          69 :                      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
    2635         207 :                                                         ExcessBits)));
    2636             :   }
    2637         156 : }
    2638             : 
    2639          73 : void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
    2640             :                                          SDValue &Lo, SDValue &Hi) {
    2641         146 :   EVT VT = N->getValueType(0);
    2642         137 :   SDLoc dl(N);
    2643         219 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2644             : 
    2645         146 :   if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
    2646          18 :     SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2647          18 :     SplitInteger(Res.getValue(1), Lo, Hi);
    2648             :     return;
    2649             :   }
    2650             : 
    2651          64 :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2652         128 :   if (VT == MVT::i16)
    2653             :     LC = RTLIB::SREM_I16;
    2654         128 :   else if (VT == MVT::i32)
    2655             :     LC = RTLIB::SREM_I32;
    2656         126 :   else if (VT == MVT::i64)
    2657             :     LC = RTLIB::SREM_I64;
    2658          17 :   else if (VT == MVT::i128)
    2659          17 :     LC = RTLIB::SREM_I128;
    2660             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
    2661             : 
    2662         128 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
    2663             : }
    2664             : 
    2665       12305 : void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
    2666             :                                              SDValue &Lo, SDValue &Hi) {
    2667       36915 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2668       24610 :   SDLoc dl(N);
    2669       36915 :   Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
    2670       49220 :   Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
    2671       24610 :                    N->getOperand(0),
    2672       12305 :                    DAG.getConstant(NVT.getSizeInBits(), dl,
    2673       73830 :                                    TLI.getPointerTy(DAG.getDataLayout())));
    2674       24610 :   Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
    2675       12305 : }
    2676             : 
    2677          11 : void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
    2678             :                                           SDValue &Lo, SDValue &Hi) {
    2679          22 :   EVT VT = N->getValueType(0);
    2680          20 :   SDLoc dl(N);
    2681             : 
    2682             :   // A divide for UMULO should be faster than a function call.
    2683          11 :   if (N->getOpcode() == ISD::UMULO) {
    2684           6 :     SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
    2685             : 
    2686           6 :     SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
    2687           2 :     SplitInteger(MUL, Lo, Hi);
    2688             : 
    2689             :     // A divide for UMULO will be faster than a function call. Select to
    2690             :     // make sure we aren't using 0.
    2691           2 :     SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
    2692           2 :                                   RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
    2693           2 :     SDValue NotZero = DAG.getSelect(dl, VT, isZero,
    2694           2 :                                     DAG.getConstant(1, dl, VT), RHS);
    2695           4 :     SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
    2696           2 :     SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
    2697           4 :                                     ISD::SETNE);
    2698           4 :     Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
    2699             :                              DAG.getConstant(0, dl, N->getValueType(1)),
    2700           6 :                              Overflow);
    2701           2 :     ReplaceValueWith(SDValue(N, 1), Overflow);
    2702             :     return;
    2703             :   }
    2704             : 
    2705           9 :   Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
    2706          36 :   EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
    2707           9 :   Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
    2708             : 
    2709             :   // Replace this with a libcall that will check overflow.
    2710           9 :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2711          18 :   if (VT == MVT::i32)
    2712             :     LC = RTLIB::MULO_I32;
    2713          18 :   else if (VT == MVT::i64)
    2714             :     LC = RTLIB::MULO_I64;
    2715           2 :   else if (VT == MVT::i128)
    2716           2 :     LC = RTLIB::MULO_I128;
    2717             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
    2718             : 
    2719           9 :   SDValue Temp = DAG.CreateStackTemporary(PtrVT);
    2720             :   // Temporary for the overflow value, default it to zero.
    2721             :   SDValue Chain =
    2722          18 :       DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
    2723          27 :                    MachinePointerInfo());
    2724             : 
    2725          18 :   TargetLowering::ArgListTy Args;
    2726           9 :   TargetLowering::ArgListEntry Entry;
    2727          54 :   for (const SDValue &Op : N->op_values()) {
    2728          18 :     EVT ArgVT = Op.getValueType();
    2729          18 :     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
    2730          18 :     Entry.Node = Op;
    2731          18 :     Entry.Ty = ArgTy;
    2732          18 :     Entry.IsSExt = true;
    2733          18 :     Entry.IsZExt = false;
    2734          18 :     Args.push_back(Entry);
    2735             :   }
    2736             : 
    2737             :   // Also pass the address of the overflow check.
    2738           9 :   Entry.Node = Temp;
    2739           9 :   Entry.Ty = PtrTy->getPointerTo();
    2740           9 :   Entry.IsSExt = true;
    2741           9 :   Entry.IsZExt = false;
    2742           9 :   Args.push_back(Entry);
    2743             : 
    2744          18 :   SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
    2745             : 
    2746          18 :   TargetLowering::CallLoweringInfo CLI(DAG);
    2747           9 :   CLI.setDebugLoc(dl)
    2748           9 :       .setChain(Chain)
    2749          18 :       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
    2750           9 :       .setSExtResult();
    2751             : 
    2752           9 :   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
    2753             : 
    2754           9 :   SplitInteger(CallInfo.first, Lo, Hi);
    2755             :   SDValue Temp2 =
    2756          27 :       DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
    2757           9 :   SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
    2758             :                              DAG.getConstant(0, dl, PtrVT),
    2759          18 :                              ISD::SETNE);
    2760             :   // Use the overflow from the libcall everywhere.
    2761           9 :   ReplaceValueWith(SDValue(N, 1), Ofl);
    2762             : }
    2763             : 
    2764          59 : void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
    2765             :                                          SDValue &Lo, SDValue &Hi) {
    2766         118 :   EVT VT = N->getValueType(0);
    2767         113 :   SDLoc dl(N);
    2768         177 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2769             : 
    2770         118 :   if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
    2771          10 :     SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2772          10 :     SplitInteger(Res.getValue(0), Lo, Hi);
    2773             :     return;
    2774             :   }
    2775             : 
    2776          54 :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2777         108 :   if (VT == MVT::i16)
    2778             :     LC = RTLIB::UDIV_I16;
    2779         108 :   else if (VT == MVT::i32)
    2780             :     LC = RTLIB::UDIV_I32;
    2781         106 :   else if (VT == MVT::i64)
    2782             :     LC = RTLIB::UDIV_I64;
    2783          18 :   else if (VT == MVT::i128)
    2784          18 :     LC = RTLIB::UDIV_I128;
    2785             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
    2786             : 
    2787         108 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
    2788             : }
    2789             : 
    2790          76 : void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
    2791             :                                          SDValue &Lo, SDValue &Hi) {
    2792         152 :   EVT VT = N->getValueType(0);
    2793         141 :   SDLoc dl(N);
    2794         228 :   SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
    2795             : 
    2796         152 :   if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
    2797          22 :     SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
    2798          22 :     SplitInteger(Res.getValue(1), Lo, Hi);
    2799             :     return;
    2800             :   }
    2801             : 
    2802          65 :   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
    2803         130 :   if (VT == MVT::i16)
    2804             :     LC = RTLIB::UREM_I16;
    2805         130 :   else if (VT == MVT::i32)
    2806             :     LC = RTLIB::UREM_I32;
    2807         128 :   else if (VT == MVT::i64)
    2808             :     LC = RTLIB::UREM_I64;
    2809          16 :   else if (VT == MVT::i128)
    2810          16 :     LC = RTLIB::UREM_I128;
    2811             :   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
    2812             : 
    2813         130 :   SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
    2814             : }
    2815             : 
    2816        5298 : void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
    2817             :                                                 SDValue &Lo, SDValue &Hi) {
    2818       15894 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2819       10596 :   SDLoc dl(N);
    2820       10596 :   SDValue Op = N->getOperand(0);
    2821       10596 :   if (Op.getValueType().bitsLE(NVT)) {
    2822             :     // The low part is zero extension of the input (degenerates to a copy).
    2823       15894 :     Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
    2824        5298 :     Hi = DAG.getConstant(0, dl, NVT);   // The high part is just a zero.
    2825             :   } else {
    2826             :     // For example, extension of an i48 to an i64.  The operand type necessarily
    2827             :     // promotes to the result type, so will end up being expanded too.
    2828             :     assert(getTypeAction(Op.getValueType()) ==
    2829             :            TargetLowering::TypePromoteInteger &&
    2830             :            "Only know how to promote this result!");
    2831           0 :     SDValue Res = GetPromotedInteger(Op);
    2832             :     assert(Res.getValueType() == N->getValueType(0) &&
    2833             :            "Operand over promoted?");
    2834             :     // Split the promoted operand.  This will simplify when it is expanded.
    2835           0 :     SplitInteger(Res, Lo, Hi);
    2836           0 :     unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
    2837           0 :     Hi = DAG.getZeroExtendInReg(Hi, dl,
    2838           0 :                                 EVT::getIntegerVT(*DAG.getContext(),
    2839           0 :                                                   ExcessBits));
    2840             :   }
    2841        5298 : }
    2842             : 
    2843          11 : void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
    2844             :                                                 SDValue &Lo, SDValue &Hi) {
    2845          22 :   SDLoc dl(N);
    2846          11 :   EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
    2847          33 :   SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
    2848          11 :   SDValue Zero = DAG.getConstant(0, dl, VT);
    2849          11 :   SDValue Swap = DAG.getAtomicCmpSwap(
    2850             :       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
    2851          33 :       cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
    2852          33 :       N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
    2853             : 
    2854          33 :   ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
    2855          33 :   ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
    2856          11 : }
    2857             : 
    2858             : //===----------------------------------------------------------------------===//
    2859             : //  Integer Operand Expansion
    2860             : //===----------------------------------------------------------------------===//
    2861             : 
    2862             : /// ExpandIntegerOperand - This method is called when the specified operand of
    2863             : /// the specified node is found to need expansion.  At this point, all of the
    2864             : /// result types of the node are known to be legal, but other operands of the
    2865             : /// node may need promotion or expansion as well as the specified one.
    2866      200617 : bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
    2867             :   DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
    2868      200617 :   SDValue Res = SDValue();
    2869             : 
    2870      601851 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    2871             :     return false;
    2872             : 
    2873      400256 :   switch (N->getOpcode()) {
    2874           0 :   default:
    2875             :   #ifndef NDEBUG
    2876             :     dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
    2877             :     N->dump(&DAG); dbgs() << "\n";
    2878             :   #endif
    2879           0 :     llvm_unreachable("Do not know how to expand this operator's operand!");
    2880             : 
    2881        3644 :   case ISD::BITCAST:           Res = ExpandOp_BITCAST(N); break;
    2882           0 :   case ISD::BR_CC:             Res = ExpandIntOp_BR_CC(N); break;
    2883       16213 :   case ISD::BUILD_VECTOR:      Res = ExpandOp_BUILD_VECTOR(N); break;
    2884       16952 :   case ISD::EXTRACT_ELEMENT:   Res = ExpandOp_EXTRACT_ELEMENT(N); break;
    2885          30 :   case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
    2886          39 :   case ISD::SCALAR_TO_VECTOR:  Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
    2887        1754 :   case ISD::SELECT_CC:         Res = ExpandIntOp_SELECT_CC(N); break;
    2888         783 :   case ISD::SETCC:             Res = ExpandIntOp_SETCC(N); break;
    2889           0 :   case ISD::SETCCE:            Res = ExpandIntOp_SETCCE(N); break;
    2890          24 :   case ISD::SETCCCARRY:        Res = ExpandIntOp_SETCCCARRY(N); break;
    2891          42 :   case ISD::SINT_TO_FP:        Res = ExpandIntOp_SINT_TO_FP(N); break;
    2892      137846 :   case ISD::STORE:   Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
    2893       89739 :   case ISD::TRUNCATE:          Res = ExpandIntOp_TRUNCATE(N); break;
    2894          39 :   case ISD::UINT_TO_FP:        Res = ExpandIntOp_UINT_TO_FP(N); break;
    2895             : 
    2896        1934 :   case ISD::SHL:
    2897             :   case ISD::SRA:
    2898             :   case ISD::SRL:
    2899             :   case ISD::ROTL:
    2900        1934 :   case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
    2901           1 :   case ISD::RETURNADDR:
    2902           1 :   case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;
    2903             : 
    2904          11 :   case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
    2905             :   }
    2906             : 
    2907             :   // If the result is null, the sub-method took care of registering results etc.
    2908      200128 :   if (!Res.getNode()) return false;
    2909             : 
    2910             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    2911             :   // core about this.
    2912      200128 :   if (Res.getNode() == N)
    2913             :     return true;
    2914             : 
    2915             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    2916             :          "Invalid operand expansion");
    2917             : 
    2918      196010 :   ReplaceValueWith(SDValue(N, 0), Res);
    2919      196010 :   return false;
    2920             : }
    2921             : 
    2922             : /// IntegerExpandSetCCOperands - Expand the operands of a comparison.  This code
    2923             : /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
    2924        2537 : void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
    2925             :                                                   SDValue &NewRHS,
    2926             :                                                   ISD::CondCode &CCCode,
    2927             :                                                   const SDLoc &dl) {
    2928        2537 :   SDValue LHSLo, LHSHi, RHSLo, RHSHi;
    2929        2537 :   GetExpandedInteger(NewLHS, LHSLo, LHSHi);
    2930        2537 :   GetExpandedInteger(NewRHS, RHSLo, RHSHi);
    2931             : 
    2932        2537 :   if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
    2933         173 :     if (RHSLo == RHSHi) {
    2934         170 :       if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
    2935         170 :         if (RHSCST->isAllOnesValue()) {
    2936             :           // Equality comparison to -1.
    2937           0 :           NewLHS = DAG.getNode(ISD::AND, dl,
    2938           0 :                                LHSLo.getValueType(), LHSLo, LHSHi);
    2939           0 :           NewRHS = RHSLo;
    2940         683 :           return;
    2941             :         }
    2942             :       }
    2943             :     }
    2944             : 
    2945         930 :     NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
    2946         930 :     NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
    2947         930 :     NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
    2948         620 :     NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
    2949         310 :     return;
    2950             :   }
    2951             : 
    2952             :   // If this is a comparison of the sign bit, just look at the top part.
    2953             :   // X > -1,  x < 0
    2954         305 :   if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
    2955         445 :     if ((CCCode == ISD::SETLT && CST->isNullValue()) ||     // X < 0
    2956          46 :         (CCCode == ISD::SETGT && CST->isAllOnesValue())) {  // X > -1
    2957         139 :       NewLHS = LHSHi;
    2958         139 :       NewRHS = RHSHi;
    2959         139 :       return;
    2960             :     }
    2961             : 
    2962             :   // FIXME: This generated code sucks.
    2963             :   ISD::CondCode LowCC;
    2964        2088 :   switch (CCCode) {
    2965           0 :   default: llvm_unreachable("Unknown integer setcc!");
    2966             :   case ISD::SETLT:
    2967             :   case ISD::SETULT: LowCC = ISD::SETULT; break;
    2968         107 :   case ISD::SETGT:
    2969         107 :   case ISD::SETUGT: LowCC = ISD::SETUGT; break;
    2970          31 :   case ISD::SETLE:
    2971          31 :   case ISD::SETULE: LowCC = ISD::SETULE; break;
    2972        1810 :   case ISD::SETGE:
    2973        1810 :   case ISD::SETUGE: LowCC = ISD::SETUGE; break;
    2974             :   }
    2975             : 
    2976             :   // LoCmp = lo(op1) < lo(op2)   // Always unsigned comparison
    2977             :   // HiCmp = hi(op1) < hi(op2)   // Signedness depends on operands
    2978             :   // dest  = hi(op1) == hi(op2) ? LoCmp : HiCmp;
    2979             : 
    2980             :   // NOTE: on targets without efficient SELECT of bools, we can always use
    2981             :   // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
    2982             :   TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
    2983        4176 :                                                  nullptr);
    2984        2088 :   SDValue LoCmp, HiCmp;
    2985        6242 :   if (TLI.isTypeLegal(LHSLo.getValueType()) &&
    2986        6198 :       TLI.isTypeLegal(RHSLo.getValueType()))
    2987        2066 :     LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
    2988        4132 :                               RHSLo, LowCC, false, DagCombineInfo, dl);
    2989        2088 :   if (!LoCmp.getNode())
    2990        4022 :     LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
    2991        4022 :                          RHSLo, LowCC);
    2992        6242 :   if (TLI.isTypeLegal(LHSHi.getValueType()) &&
    2993        6198 :       TLI.isTypeLegal(RHSHi.getValueType()))
    2994        2066 :     HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
    2995        4132 :                               RHSHi, CCCode, false, DagCombineInfo, dl);
    2996        2088 :   if (!HiCmp.getNode())
    2997        1958 :     HiCmp =
    2998        5874 :         DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
    2999        3916 :                     LHSHi, RHSHi, DAG.getCondCode(CCCode));
    3000             : 
    3001        4176 :   ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
    3002        4176 :   ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
    3003             : 
    3004        4103 :   bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
    3005        2347 :                     CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
    3006             : 
    3007        2101 :   if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
    3008         285 :       (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
    3009           2 :                       (LoCmpC && LoCmpC->isNullValue())))) {
    3010             :     // For LE / GE, if high part is known false, ignore the low part.
    3011             :     // For LT / GT: if low part is known false, return the high part.
    3012             :     //              if high part is known true, ignore the low part.
    3013           4 :     NewLHS = HiCmp;
    3014           4 :     NewRHS = SDValue();
    3015           4 :     return;
    3016             :   }
    3017             : 
    3018        2084 :   if (LHSHi == RHSHi) {
    3019             :     // Comparing the low bits is enough.
    3020           8 :     NewLHS = LoCmp;
    3021           8 :     NewRHS = SDValue();
    3022           8 :     return;
    3023             :   }
    3024             : 
    3025             :   // Lower with SETCCE or SETCCCARRY if the target supports it.
    3026        4152 :   EVT HiVT = LHSHi.getValueType();
    3027        2076 :   EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
    3028        2076 :   bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
    3029             : 
    3030             :   // FIXME: Make all targets support this, then remove the other lowering.
    3031        1953 :   if (HasSETCCCARRY ||
    3032        3906 :       TLI.getOperationAction(ISD::SETCCE, ExpandVT) == TargetLowering::Custom) {
    3033             :     // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip
    3034             :     // operands and condition code.
    3035         222 :     bool FlipOperands = false;
    3036         222 :     switch (CCCode) {
    3037          29 :     case ISD::SETGT:  CCCode = ISD::SETLT;  FlipOperands = true; break;
    3038          45 :     case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
    3039          13 :     case ISD::SETLE:  CCCode = ISD::SETGE;  FlipOperands = true; break;
    3040          11 :     case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
    3041             :     default: break;
    3042             :     }
    3043             :     if (FlipOperands) {
    3044          98 :       std::swap(LHSLo, RHSLo);
    3045             :       std::swap(LHSHi, RHSHi);
    3046             :     }
    3047             :     // Perform a wide subtraction, feeding the carry from the low part into
    3048             :     // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially
    3049             :     // looking at the high part of the result of LHS - RHS. It is negative
    3050             :     // iff LHS < RHS. It is zero or positive iff LHS >= RHS.
    3051         444 :     EVT LoVT = LHSLo.getValueType();
    3052         222 :     SDVTList VTList = DAG.getVTList(
    3053         321 :         LoVT, HasSETCCCARRY ? getSetCCResultType(LoVT) : MVT::Glue);
    3054         222 :     SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? ISD::USUBO : ISD::SUBC, dl,
    3055         222 :                                  VTList, LHSLo, RHSLo);
    3056         222 :     SDValue Res = DAG.getNode(HasSETCCCARRY ? ISD::SETCCCARRY : ISD::SETCCE, dl,
    3057             :                               getSetCCResultType(HiVT), LHSHi, RHSHi,
    3058         444 :                               LowCmp.getValue(1), DAG.getCondCode(CCCode));
    3059         222 :     NewLHS = Res;
    3060         222 :     NewRHS = SDValue();
    3061             :     return;
    3062             :   }
    3063             : 
    3064        1854 :   NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
    3065        1854 :                              false, DagCombineInfo, dl);
    3066        1854 :   if (!NewLHS.getNode())
    3067        1827 :     NewLHS =
    3068        3654 :         DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
    3069        3708 :   NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
    3070        1854 :   NewRHS = SDValue();
    3071             : }
    3072             : 
    3073           0 : SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
    3074           0 :   SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
    3075           0 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
    3076           0 :   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
    3077             : 
    3078             :   // If ExpandSetCCOperands returned a scalar, we need to compare the result
    3079             :   // against zero to select between true and false values.
    3080           0 :   if (!NewRHS.getNode()) {
    3081           0 :     NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
    3082           0 :     CCCode = ISD::SETNE;
    3083             :   }
    3084             : 
    3085             :   // Update N to have the operands specified.
    3086           0 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
    3087             :                                 DAG.getCondCode(CCCode), NewLHS, NewRHS,
    3088           0 :                                 N->getOperand(4)), 0);
    3089             : }
    3090             : 
    3091        1754 : SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
    3092        5262 :   SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
    3093        5262 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
    3094        3508 :   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
    3095             : 
    3096             :   // If ExpandSetCCOperands returned a scalar, we need to compare the result
    3097             :   // against zero to select between true and false values.
    3098        1754 :   if (!NewRHS.getNode()) {
    3099        6936 :     NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
    3100        1734 :     CCCode = ISD::SETNE;
    3101             :   }
    3102             : 
    3103             :   // Update N to have the operands specified.
    3104        3508 :   return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
    3105        5262 :                                 N->getOperand(2), N->getOperand(3),
    3106        1754 :                                 DAG.getCondCode(CCCode)), 0);
    3107             : }
    3108             : 
    3109         783 : SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
    3110        2349 :   SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
    3111        2349 :   ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
    3112        1566 :   IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
    3113             : 
    3114             :   // If ExpandSetCCOperands returned a scalar, use it.
    3115         783 :   if (!NewRHS.getNode()) {
    3116             :     assert(NewLHS.getValueType() == N->getValueType(0) &&
    3117             :            "Unexpected setcc expansion!");
    3118         354 :     return NewLHS;
    3119             :   }
    3120             : 
    3121             :   // Otherwise, update N to have the operands specified.
    3122         429 :   return SDValue(
    3123         858 :       DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
    3124             : }
    3125             : 
    3126           0 : SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) {
    3127           0 :   SDValue LHS = N->getOperand(0);
    3128           0 :   SDValue RHS = N->getOperand(1);
    3129           0 :   SDValue Carry = N->getOperand(2);
    3130           0 :   SDValue Cond = N->getOperand(3);
    3131           0 :   SDLoc dl = SDLoc(N);
    3132             : 
    3133           0 :   SDValue LHSLo, LHSHi, RHSLo, RHSHi;
    3134           0 :   GetExpandedInteger(LHS, LHSLo, LHSHi);
    3135           0 :   GetExpandedInteger(RHS, RHSLo, RHSHi);
    3136             : 
    3137             :   // Expand to a SUBE for the low part and a smaller SETCCE for the high.
    3138           0 :   SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
    3139           0 :   SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry);
    3140           0 :   return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi,
    3141           0 :                      LowCmp.getValue(1), Cond);
    3142             : }
    3143             : 
    3144          24 : SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
    3145          48 :   SDValue LHS = N->getOperand(0);
    3146          48 :   SDValue RHS = N->getOperand(1);
    3147          48 :   SDValue Carry = N->getOperand(2);
    3148          48 :   SDValue Cond = N->getOperand(3);
    3149          48 :   SDLoc dl = SDLoc(N);
    3150             : 
    3151          24 :   SDValue LHSLo, LHSHi, RHSLo, RHSHi;
    3152          24 :   GetExpandedInteger(LHS, LHSLo, LHSHi);
    3153          24 :   GetExpandedInteger(RHS, RHSLo, RHSHi);
    3154             : 
    3155             :   // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
    3156          72 :   SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
    3157          24 :   SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
    3158          24 :   return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
    3159          96 :                      LowCmp.getValue(1), Cond);
    3160             : }
    3161             : 
    3162        1934 : SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
    3163             :   // The value being shifted is legal, but the shift amount is too big.
    3164             :   // It follows that either the result of the shift is undefined, or the
    3165             :   // upper half of the shift amount is zero.  Just use the lower half.
    3166        1934 :   SDValue Lo, Hi;
    3167        3868 :   GetExpandedInteger(N->getOperand(1), Lo, Hi);
    3168        3868 :   return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
    3169             : }
    3170             : 
    3171           1 : SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
    3172             :   // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant.  This
    3173             :   // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
    3174             :   // constant to valid type.
    3175           1 :   SDValue Lo, Hi;
    3176           2 :   GetExpandedInteger(N->getOperand(0), Lo, Hi);
    3177           1 :   return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
    3178             : }
    3179             : 
    3180          42 : SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
    3181          84 :   SDValue Op = N->getOperand(0);
    3182          84 :   EVT DstVT = N->getValueType(0);
    3183          84 :   RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
    3184             :   assert(LC != RTLIB::UNKNOWN_LIBCALL &&
    3185             :          "Don't know how to expand this SINT_TO_FP!");
    3186         168 :   return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
    3187             : }
    3188             : 
    3189       68923 : SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
    3190       67539 :   if (ISD::isNormalStore(N))
    3191       67539 :     return ExpandOp_NormalStore(N, OpNo);
    3192             : 
    3193             :   assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
    3194             :   assert(OpNo == 1 && "Can only expand the stored value so far");
    3195             : 
    3196        4152 :   EVT VT = N->getOperand(1).getValueType();
    3197        2768 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3198        2768 :   SDValue Ch  = N->getChain();
    3199        1384 :   SDValue Ptr = N->getBasePtr();
    3200        2768 :   unsigned Alignment = N->getAlignment();
    3201        1384 :   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
    3202        2768 :   AAMDNodes AAInfo = N->getAAInfo();
    3203        1384 :   SDLoc dl(N);
    3204        1384 :   SDValue Lo, Hi;
    3205             : 
    3206             :   assert(NVT.isByteSized() && "Expanded type not byte sized!");
    3207             : 
    3208        1384 :   if (N->getMemoryVT().bitsLE(NVT)) {
    3209         336 :     GetExpandedInteger(N->getValue(), Lo, Hi);
    3210         672 :     return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
    3211         672 :                              N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
    3212             :   }
    3213             : 
    3214        2096 :   if (DAG.getDataLayout().isLittleEndian()) {
    3215             :     // Little-endian - low bits are at low addresses.
    3216        1042 :     GetExpandedInteger(N->getValue(), Lo, Hi);
    3217             : 
    3218        3126 :     Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
    3219        1042 :                       AAInfo);
    3220             : 
    3221             :     unsigned ExcessBits =
    3222        1042 :       N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
    3223        1042 :     EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
    3224             : 
    3225             :     // Increment the pointer to the other half.
    3226        1042 :     unsigned IncrementSize = NVT.getSizeInBits()/8;
    3227        2084 :     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
    3228        3126 :                       DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
    3229        2084 :     Hi = DAG.getTruncStore(
    3230        1042 :         Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
    3231        4168 :         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    3232        3126 :     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
    3233             :   }
    3234             : 
    3235             :   // Big-endian - high bits are at low addresses.  Favor aligned stores at
    3236             :   // the cost of some bit-fiddling.
    3237           6 :   GetExpandedInteger(N->getValue(), Lo, Hi);
    3238             : 
    3239           6 :   EVT ExtVT = N->getMemoryVT();
    3240           6 :   unsigned EBytes = ExtVT.getStoreSize();
    3241           6 :   unsigned IncrementSize = NVT.getSizeInBits()/8;
    3242           6 :   unsigned ExcessBits = (EBytes - IncrementSize)*8;
    3243           6 :   EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
    3244          12 :                                ExtVT.getSizeInBits() - ExcessBits);
    3245             : 
    3246           6 :   if (ExcessBits < NVT.getSizeInBits()) {
    3247             :     // Transfer high bits from the top of Lo to the bottom of Hi.
    3248          10 :     Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
    3249           5 :                      DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
    3250          30 :                                      TLI.getPointerTy(DAG.getDataLayout())));
    3251          10 :     Hi = DAG.getNode(
    3252             :         ISD::OR, dl, NVT, Hi,
    3253           5 :         DAG.getNode(ISD::SRL, dl, NVT, Lo,
    3254             :                     DAG.getConstant(ExcessBits, dl,
    3255          30 :                                     TLI.getPointerTy(DAG.getDataLayout()))));
    3256             :   }
    3257             : 
    3258             :   // Store both the high bits and maybe some of the low bits.
    3259          18 :   Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
    3260           6 :                          MMOFlags, AAInfo);
    3261             : 
    3262             :   // Increment the pointer to the other half.
    3263          12 :   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
    3264          18 :                     DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
    3265             :   // Store the lowest ExcessBits bits in the second half.
    3266          12 :   Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
    3267           6 :                          N->getPointerInfo().getWithOffset(IncrementSize),
    3268           6 :                          EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
    3269          30 :                          MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
    3270          18 :   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
    3271             : }
    3272             : 
    3273       89739 : SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
    3274       89739 :   SDValue InL, InH;
    3275      179478 :   GetExpandedInteger(N->getOperand(0), InL, InH);
    3276             :   // Just truncate the low part of the source.
    3277      448695 :   return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
    3278             : }
    3279             : 
    3280          39 : SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
    3281          78 :   SDValue Op = N->getOperand(0);
    3282          78 :   EVT SrcVT = Op.getValueType();
    3283          78 :   EVT DstVT = N->getValueType(0);
    3284          78 :   SDLoc dl(N);
    3285             : 
    3286             :   // The following optimization is valid only if every value in SrcVT (when
    3287             :   // treated as signed) is representable in DstVT.  Check that the mantissa
    3288             :   // size of DstVT is >= than the number of bits in SrcVT -1.
    3289          39 :   const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
    3290          39 :   if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
    3291           0 :       TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){
    3292             :     // Do a signed conversion then adjust the result.
    3293           0 :     SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
    3294           0 :     SignedConv = TLI.LowerOperation(SignedConv, DAG);
    3295             : 
    3296             :     // The result of the signed conversion needs adjusting if the 'sign bit' of
    3297             :     // the incoming integer was set.  To handle this, we dynamically test to see
    3298             :     // if it is set, and, if so, add a fudge factor.
    3299             : 
    3300           0 :     const uint64_t F32TwoE32  = 0x4F800000ULL;
    3301           0 :     const uint64_t F32TwoE64  = 0x5F800000ULL;
    3302           0 :     const uint64_t F32TwoE128 = 0x7F800000ULL;
    3303             : 
    3304           0 :     APInt FF(32, 0);
    3305           0 :     if (SrcVT == MVT::i32)
    3306           0 :       FF = APInt(32, F32TwoE32);
    3307           0 :     else if (SrcVT == MVT::i64)
    3308           0 :       FF = APInt(32, F32TwoE64);
    3309           0 :     else if (SrcVT == MVT::i128)
    3310           0 :       FF = APInt(32, F32TwoE128);
    3311             :     else
    3312           0 :       llvm_unreachable("Unsupported UINT_TO_FP!");
    3313             : 
    3314             :     // Check whether the sign bit is set.
    3315           0 :     SDValue Lo, Hi;
    3316           0 :     GetExpandedInteger(Op, Lo, Hi);
    3317           0 :     SDValue SignSet = DAG.getSetCC(dl,
    3318             :                                    getSetCCResultType(Hi.getValueType()),
    3319             :                                    Hi,
    3320             :                                    DAG.getConstant(0, dl, Hi.getValueType()),
    3321           0 :                                    ISD::SETLT);
    3322             : 
    3323             :     // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
    3324             :     SDValue FudgePtr =
    3325           0 :         DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
    3326           0 :                             TLI.getPointerTy(DAG.getDataLayout()));
    3327             : 
    3328             :     // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
    3329           0 :     SDValue Zero = DAG.getIntPtrConstant(0, dl);
    3330           0 :     SDValue Four = DAG.getIntPtrConstant(4, dl);
    3331           0 :     if (DAG.getDataLayout().isBigEndian())
    3332             :       std::swap(Zero, Four);
    3333           0 :     SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
    3334           0 :                                    Zero, Four);
    3335           0 :     unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
    3336           0 :     FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
    3337           0 :                            FudgePtr, Offset);
    3338           0 :     Alignment = std::min(Alignment, 4u);
    3339             : 
    3340             :     // Load the value out, extending it from f32 to the destination float type.
    3341             :     // FIXME: Avoid the extend by constructing the right constant pool?
    3342           0 :     SDValue Fudge = DAG.getExtLoad(
    3343           0 :         ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
    3344             :         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
    3345           0 :         Alignment);
    3346           0 :     return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
    3347             :   }
    3348             : 
    3349             :   // Otherwise, use a libcall.
    3350          39 :   RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
    3351             :   assert(LC != RTLIB::UNKNOWN_LIBCALL &&
    3352             :          "Don't know how to expand this UINT_TO_FP!");
    3353          78 :   return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
    3354             : }
    3355             : 
    3356          11 : SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
    3357          22 :   SDLoc dl(N);
    3358          11 :   SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
    3359          11 :                                cast<AtomicSDNode>(N)->getMemoryVT(),
    3360          22 :                                N->getOperand(0),
    3361          33 :                                N->getOperand(1), N->getOperand(2),
    3362          22 :                                cast<AtomicSDNode>(N)->getMemOperand());
    3363          33 :   return Swap.getValue(1);
    3364             : }
    3365             : 
    3366             : 
    3367        2520 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
    3368        5040 :   SDValue InOp0 = N->getOperand(0);
    3369        5040 :   EVT InVT = InOp0.getValueType();
    3370             : 
    3371        5040 :   EVT OutVT = N->getValueType(0);
    3372        5040 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3373             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3374        2520 :   unsigned OutNumElems = OutVT.getVectorNumElements();
    3375        2520 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3376             : 
    3377        5040 :   SDLoc dl(N);
    3378        5040 :   SDValue BaseIdx = N->getOperand(1);
    3379             : 
    3380        5040 :   SmallVector<SDValue, 8> Ops;
    3381        2520 :   Ops.reserve(OutNumElems);
    3382       10434 :   for (unsigned i = 0; i != OutNumElems; ++i) {
    3383             : 
    3384             :     // Extract the element from the original vector.
    3385        7914 :     SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
    3386       23742 :       BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
    3387        7914 :     SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
    3388       15828 :       InVT.getVectorElementType(), N->getOperand(0), Index);
    3389             : 
    3390       15828 :     SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
    3391             :     // Insert the converted element to the new vector.
    3392        7914 :     Ops.push_back(Op);
    3393             :   }
    3394             : 
    3395        7560 :   return DAG.getBuildVector(NOutVT, dl, Ops);
    3396             : }
    3397             : 
    3398             : 
    3399          84 : SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
    3400          84 :   ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
    3401         168 :   EVT VT = N->getValueType(0);
    3402         168 :   SDLoc dl(N);
    3403             : 
    3404         168 :   ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
    3405             : 
    3406         168 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3407         168 :   SDValue V1 = GetPromotedInteger(N->getOperand(1));
    3408         168 :   EVT OutVT = V0.getValueType();
    3409             : 
    3410         168 :   return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
    3411             : }
    3412             : 
    3413             : 
    3414        1644 : SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
    3415        3288 :   EVT OutVT = N->getValueType(0);
    3416        3288 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3417             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3418        3288 :   unsigned NumElems = N->getNumOperands();
    3419        1644 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3420             : 
    3421        3288 :   SDLoc dl(N);
    3422             : 
    3423        3288 :   SmallVector<SDValue, 8> Ops;
    3424        1644 :   Ops.reserve(NumElems);
    3425       15762 :   for (unsigned i = 0; i != NumElems; ++i) {
    3426       14118 :     SDValue Op;
    3427             :     // BUILD_VECTOR integer operand types are allowed to be larger than the
    3428             :     // result's element type. This may still be true after the promotion. For
    3429             :     // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
    3430             :     // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
    3431       42354 :     if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
    3432       29070 :       Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
    3433             :     else
    3434        8856 :       Op = N->getOperand(i);
    3435       14118 :     Ops.push_back(Op);
    3436             :   }
    3437             : 
    3438        4932 :   return DAG.getBuildVector(NOutVT, dl, Ops);
    3439             : }
    3440             : 
    3441          26 : SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
    3442             : 
    3443          52 :   SDLoc dl(N);
    3444             : 
    3445             :   assert(!N->getOperand(0).getValueType().isVector() &&
    3446             :          "Input must be a scalar");
    3447             : 
    3448          52 :   EVT OutVT = N->getValueType(0);
    3449          52 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3450             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3451          26 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3452             : 
    3453          78 :   SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
    3454             : 
    3455          78 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
    3456             : }
    3457             : 
    3458        1632 : SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
    3459        3264 :   SDLoc dl(N);
    3460             : 
    3461        3264 :   EVT OutVT = N->getValueType(0);
    3462        3264 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3463             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3464             : 
    3465        1632 :   EVT InElemTy = OutVT.getVectorElementType();
    3466        1632 :   EVT OutElemTy = NOutVT.getVectorElementType();
    3467             : 
    3468        4896 :   unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
    3469        1632 :   unsigned NumOutElem = NOutVT.getVectorNumElements();
    3470        3264 :   unsigned NumOperands = N->getNumOperands();
    3471             :   assert(NumElem * NumOperands == NumOutElem &&
    3472             :          "Unexpected number of elements");
    3473             : 
    3474             :   // Take the elements from the first vector.
    3475        4896 :   SmallVector<SDValue, 8> Ops(NumOutElem);
    3476        6810 :   for (unsigned i = 0; i < NumOperands; ++i) {
    3477       10356 :     SDValue Op = N->getOperand(i);
    3478       29902 :     for (unsigned j = 0; j < NumElem; ++j) {
    3479       24724 :       SDValue Ext = DAG.getNode(
    3480             :           ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op,
    3481       74172 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3482       74172 :       Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
    3483             :     }
    3484             :   }
    3485             : 
    3486        4896 :   return DAG.getBuildVector(NOutVT, dl, Ops);
    3487             : }
    3488             : 
    3489           3 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
    3490           6 :   EVT VT = N->getValueType(0);
    3491           6 :   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3492             :   assert(NVT.isVector() && "This type must be promoted to a vector type");
    3493             : 
    3494           6 :   SDLoc dl(N);
    3495             : 
    3496             :   // For operands whose TypeAction is to promote, extend the promoted node
    3497             :   // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
    3498             :   // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
    3499             :   // type..
    3500          12 :   if (getTypeAction(N->getOperand(0).getValueType())
    3501             :       == TargetLowering::TypePromoteInteger) {
    3502           3 :     SDValue Promoted;
    3503             : 
    3504           6 :     switch(N->getOpcode()) {
    3505           1 :       case ISD::SIGN_EXTEND_VECTOR_INREG:
    3506           2 :         Promoted = SExtPromotedInteger(N->getOperand(0));
    3507           1 :         break;
    3508           1 :       case ISD::ZERO_EXTEND_VECTOR_INREG:
    3509           2 :         Promoted = ZExtPromotedInteger(N->getOperand(0));
    3510           1 :         break;
    3511           1 :       case ISD::ANY_EXTEND_VECTOR_INREG:
    3512           2 :         Promoted = GetPromotedInteger(N->getOperand(0));
    3513           1 :         break;
    3514           0 :       default:
    3515           0 :         llvm_unreachable("Node has unexpected Opcode");
    3516             :     }
    3517           9 :     return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
    3518             :   }
    3519             : 
    3520             :   // Directly extend to the appropriate transform-to type.
    3521           0 :   return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
    3522             : }
    3523             : 
    3524         182 : SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
    3525         364 :   EVT OutVT = N->getValueType(0);
    3526         364 :   EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
    3527             :   assert(NOutVT.isVector() && "This type must be promoted to a vector type");
    3528             : 
    3529         182 :   EVT NOutVTElem = NOutVT.getVectorElementType();
    3530             : 
    3531         364 :   SDLoc dl(N);
    3532         364 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3533             : 
    3534         182 :   SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
    3535         364 :     NOutVTElem, N->getOperand(1));
    3536         182 :   return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
    3537         546 :     V0, ConvElem, N->getOperand(2));
    3538             : }
    3539             : 
    3540       14880 : SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    3541       29760 :   SDLoc dl(N);
    3542       29760 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3543       44640 :   SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
    3544       59520 :                                   TLI.getVectorIdxTy(DAG.getDataLayout()));
    3545       14880 :   SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
    3546       29760 :     V0->getValueType(0).getScalarType(), V0, V1);
    3547             : 
    3548             :   // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
    3549             :   // element types. If this is the case then we need to expand the outgoing
    3550             :   // value and not truncate it.
    3551       44640 :   return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
    3552             : }
    3553             : 
    3554          42 : SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
    3555          84 :   SDLoc dl(N);
    3556          84 :   SDValue V0 = GetPromotedInteger(N->getOperand(0));
    3557          84 :   MVT InVT = V0.getValueType().getSimpleVT();
    3558             :   MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
    3559          84 :                                N->getValueType(0).getVectorNumElements());
    3560         168 :   SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
    3561         168 :   return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
    3562             : }
    3563             : 
    3564        1678 : SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
    3565        3356 :   SDLoc dl(N);
    3566        3356 :   unsigned NumElems = N->getNumOperands();
    3567             : 
    3568        3356 :   EVT RetSclrTy = N->getValueType(0).getVectorElementType();
    3569             : 
    3570        3356 :   SmallVector<SDValue, 8> NewOps;
    3571        1678 :   NewOps.reserve(NumElems);
    3572             : 
    3573             :   // For each incoming vector
    3574        6674 :   for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
    3575        9992 :     SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
    3576        9992 :     EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
    3577        9992 :     unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
    3578             : 
    3579       24244 :     for (unsigned i=0; i<NumElem; ++i) {
    3580             :       // Extract element from incoming vector
    3581       19248 :       SDValue Ex = DAG.getNode(
    3582             :           ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
    3583       57744 :           DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3584       38496 :       SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
    3585       19248 :       NewOps.push_back(Tr);
    3586             :     }
    3587             :   }
    3588             : 
    3589        6712 :   return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
    3590             : }

Generated by: LCOV version 1.13