LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeVectorTypes.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1958 2246 87.2 %
Date: 2017-09-14 15:23:50 Functions: 108 118 91.5 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file performs vector type splitting and scalarization for LegalizeTypes.
      11             : // Scalarization is the act of changing a computation in an illegal one-element
      12             : // vector type to be a computation in its scalar element type.  For example,
      13             : // implementing <1 x f32> arithmetic in a scalar f32 register.  This is needed
      14             : // as a base case when scalarizing vector arithmetic like <4 x f32>, which
      15             : // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
      16             : // types.
      17             : // Splitting is the act of changing a computation in an invalid vector type to
      18             : // be a computation in two vectors of half the size.  For example, implementing
      19             : // <128 x f32> operations in terms of two <64 x f32> operations.
      20             : //
      21             : //===----------------------------------------------------------------------===//
      22             : 
      23             : #include "LegalizeTypes.h"
      24             : #include "llvm/IR/DataLayout.h"
      25             : #include "llvm/Support/ErrorHandling.h"
      26             : #include "llvm/Support/raw_ostream.h"
      27             : using namespace llvm;
      28             : 
      29             : #define DEBUG_TYPE "legalize-types"
      30             : 
      31             : //===----------------------------------------------------------------------===//
      32             : //  Result Vector Scalarization: <1 x ty> -> ty.
      33             : //===----------------------------------------------------------------------===//
      34             : 
      35       52778 : void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
      36             :   DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
      37             :         N->dump(&DAG);
      38             :         dbgs() << "\n");
      39       52778 :   SDValue R = SDValue();
      40             : 
      41      105556 :   switch (N->getOpcode()) {
      42           0 :   default:
      43             : #ifndef NDEBUG
      44             :     dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
      45             :     N->dump(&DAG);
      46             :     dbgs() << "\n";
      47             : #endif
      48           0 :     report_fatal_error("Do not know how to scalarize the result of this "
      49             :                        "operator!\n");
      50             : 
      51           0 :   case ISD::MERGE_VALUES:      R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
      52       16377 :   case ISD::BITCAST:           R = ScalarizeVecRes_BITCAST(N); break;
      53        6605 :   case ISD::BUILD_VECTOR:      R = ScalarizeVecRes_BUILD_VECTOR(N); break;
      54        4620 :   case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
      55         646 :   case ISD::FP_ROUND:          R = ScalarizeVecRes_FP_ROUND(N); break;
      56           0 :   case ISD::FP_ROUND_INREG:    R = ScalarizeVecRes_InregOp(N); break;
      57           0 :   case ISD::FPOWI:             R = ScalarizeVecRes_FPOWI(N); break;
      58         168 :   case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
      59        7934 :   case ISD::LOAD:           R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
      60         158 :   case ISD::SCALAR_TO_VECTOR:  R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
      61          19 :   case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
      62         153 :   case ISD::VSELECT:           R = ScalarizeVecRes_VSELECT(N); break;
      63         197 :   case ISD::SELECT:            R = ScalarizeVecRes_SELECT(N); break;
      64           0 :   case ISD::SELECT_CC:         R = ScalarizeVecRes_SELECT_CC(N); break;
      65         333 :   case ISD::SETCC:             R = ScalarizeVecRes_SETCC(N); break;
      66         137 :   case ISD::UNDEF:             R = ScalarizeVecRes_UNDEF(N); break;
      67           0 :   case ISD::VECTOR_SHUFFLE:    R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
      68          20 :   case ISD::ANY_EXTEND_VECTOR_INREG:
      69             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
      70             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
      71          20 :     R = ScalarizeVecRes_VecInregOp(N);
      72          20 :     break;
      73       16886 :   case ISD::ANY_EXTEND:
      74             :   case ISD::BITREVERSE:
      75             :   case ISD::BSWAP:
      76             :   case ISD::CTLZ:
      77             :   case ISD::CTLZ_ZERO_UNDEF:
      78             :   case ISD::CTPOP:
      79             :   case ISD::CTTZ:
      80             :   case ISD::CTTZ_ZERO_UNDEF:
      81             :   case ISD::FABS:
      82             :   case ISD::FCEIL:
      83             :   case ISD::FCOS:
      84             :   case ISD::FEXP:
      85             :   case ISD::FEXP2:
      86             :   case ISD::FFLOOR:
      87             :   case ISD::FLOG:
      88             :   case ISD::FLOG10:
      89             :   case ISD::FLOG2:
      90             :   case ISD::FNEARBYINT:
      91             :   case ISD::FNEG:
      92             :   case ISD::FP_EXTEND:
      93             :   case ISD::FP_TO_SINT:
      94             :   case ISD::FP_TO_UINT:
      95             :   case ISD::FRINT:
      96             :   case ISD::FROUND:
      97             :   case ISD::FSIN:
      98             :   case ISD::FSQRT:
      99             :   case ISD::FTRUNC:
     100             :   case ISD::SIGN_EXTEND:
     101             :   case ISD::SINT_TO_FP:
     102             :   case ISD::TRUNCATE:
     103             :   case ISD::UINT_TO_FP:
     104             :   case ISD::ZERO_EXTEND:
     105             :   case ISD::FCANONICALIZE:
     106       16886 :     R = ScalarizeVecRes_UnaryOp(N);
     107       16886 :     break;
     108             : 
     109        2468 :   case ISD::ADD:
     110             :   case ISD::AND:
     111             :   case ISD::FADD:
     112             :   case ISD::FCOPYSIGN:
     113             :   case ISD::FDIV:
     114             :   case ISD::FMUL:
     115             :   case ISD::FMINNUM:
     116             :   case ISD::FMAXNUM:
     117             :   case ISD::FMINNAN:
     118             :   case ISD::FMAXNAN:
     119             :   case ISD::SMIN:
     120             :   case ISD::SMAX:
     121             :   case ISD::UMIN:
     122             :   case ISD::UMAX:
     123             : 
     124             :   case ISD::FPOW:
     125             :   case ISD::FREM:
     126             :   case ISD::FSUB:
     127             :   case ISD::MUL:
     128             :   case ISD::OR:
     129             :   case ISD::SDIV:
     130             :   case ISD::SREM:
     131             :   case ISD::SUB:
     132             :   case ISD::UDIV:
     133             :   case ISD::UREM:
     134             :   case ISD::XOR:
     135             :   case ISD::SHL:
     136             :   case ISD::SRA:
     137             :   case ISD::SRL:
     138        2468 :     R = ScalarizeVecRes_BinOp(N);
     139        2468 :     break;
     140          24 :   case ISD::FMA:
     141          24 :     R = ScalarizeVecRes_TernaryOp(N);
     142          24 :     break;
     143             :   }
     144             : 
     145             :   // If R is null, the sub-method took care of registering the result.
     146       52778 :   if (R.getNode())
     147       52778 :     SetScalarizedVector(SDValue(N, ResNo), R);
     148       52778 : }
     149             : 
     150        2468 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
     151        4936 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     152        4936 :   SDValue RHS = GetScalarizedVector(N->getOperand(1));
     153        7404 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     154       14808 :                      LHS.getValueType(), LHS, RHS, N->getFlags());
     155             : }
     156             : 
     157          24 : SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
     158          48 :   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
     159          48 :   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
     160          48 :   SDValue Op2 = GetScalarizedVector(N->getOperand(2));
     161          72 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     162         120 :                      Op0.getValueType(), Op0, Op1, Op2);
     163             : }
     164             : 
     165           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
     166             :                                                        unsigned ResNo) {
     167           0 :   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
     168           0 :   return GetScalarizedVector(Op);
     169             : }
     170             : 
     171       16377 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
     172       32754 :   EVT NewVT = N->getValueType(0).getVectorElementType();
     173       49131 :   return DAG.getNode(ISD::BITCAST, SDLoc(N),
     174       65508 :                      NewVT, N->getOperand(0));
     175             : }
     176             : 
     177        6605 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
     178       13210 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     179       13210 :   SDValue InOp = N->getOperand(0);
     180             :   // The BUILD_VECTOR operands may be of wider element types and
     181             :   // we may need to truncate them back to the requested return type.
     182        6605 :   if (EltVT.isInteger())
     183       19896 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
     184        1631 :   return InOp;
     185             : }
     186             : 
     187        4620 : SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
     188       13860 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
     189       13860 :                      N->getValueType(0).getVectorElementType(),
     190       27720 :                      N->getOperand(0), N->getOperand(1));
     191             : }
     192             : 
     193         646 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
     194        1292 :   EVT NewVT = N->getValueType(0).getVectorElementType();
     195        1292 :   SDValue Op = GetScalarizedVector(N->getOperand(0));
     196        1938 :   return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
     197        2584 :                      NewVT, Op, N->getOperand(1));
     198             : }
     199             : 
     200           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
     201           0 :   SDValue Op = GetScalarizedVector(N->getOperand(0));
     202           0 :   return DAG.getNode(ISD::FPOWI, SDLoc(N),
     203           0 :                      Op.getValueType(), Op, N->getOperand(1));
     204             : }
     205             : 
     206         168 : SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
     207             :   // The value to insert may have a wider type than the vector element type,
     208             :   // so be sure to truncate it to the element type if necessary.
     209         336 :   SDValue Op = N->getOperand(1);
     210         336 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     211         360 :   if (Op.getValueType() != EltVT)
     212             :     // FIXME: Can this happen for floating point types?
     213           0 :     Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
     214         168 :   return Op;
     215             : }
     216             : 
     217        3967 : SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
     218             :   assert(N->isUnindexed() && "Indexed vector load?");
     219             : 
     220        3967 :   SDValue Result = DAG.getLoad(
     221             :       ISD::UNINDEXED, N->getExtensionType(),
     222       23802 :       N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
     223       11901 :       N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
     224       11901 :       N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
     225        3967 :       N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
     226       27769 :       N->getAAInfo());
     227             : 
     228             :   // Legalize the chain result - switch anything that used the old chain to
     229             :   // use the new one.
     230       11901 :   ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
     231        3967 :   return Result;
     232             : }
     233             : 
     234       16886 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
     235             :   // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
     236       33772 :   EVT DestVT = N->getValueType(0).getVectorElementType();
     237       33772 :   SDValue Op = N->getOperand(0);
     238       33772 :   EVT OpVT = Op.getValueType();
     239       33772 :   SDLoc DL(N);
     240             :   // The result needs scalarizing, but it's not a given that the source does.
     241             :   // This is a workaround for targets where it's impossible to scalarize the
     242             :   // result of a conversion, because the source type is legal.
     243             :   // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
     244             :   // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
     245             :   // legal and was not scalarized.
     246             :   // See the similar logic in ScalarizeVecRes_VSETCC
     247       33772 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     248       16880 :     Op = GetScalarizedVector(Op);
     249             :   } else {
     250           6 :     EVT VT = OpVT.getVectorElementType();
     251          12 :     Op = DAG.getNode(
     252             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
     253          24 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     254             :   }
     255      101316 :   return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
     256             : }
     257             : 
     258          19 : SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
     259          38 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     260          57 :   EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
     261          38 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     262          57 :   return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
     263          76 :                      LHS, DAG.getValueType(ExtVT));
     264             : }
     265             : 
     266          20 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
     267          40 :   SDLoc DL(N);
     268          40 :   SDValue Op = N->getOperand(0);
     269             : 
     270          40 :   EVT OpVT = Op.getValueType();
     271          20 :   EVT OpEltVT = OpVT.getVectorElementType();
     272          40 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     273             : 
     274          40 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     275           0 :     Op = GetScalarizedVector(Op);
     276             :   } else {
     277          40 :     Op = DAG.getNode(
     278             :         ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
     279          80 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     280             :   }
     281             : 
     282          40 :   switch (N->getOpcode()) {
     283          20 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     284          40 :     return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
     285           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     286           0 :     return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
     287           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     288           0 :     return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
     289             :   }
     290             : 
     291           0 :   llvm_unreachable("Illegal extend_vector_inreg opcode");
     292             : }
     293             : 
     294         158 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
     295             :   // If the operand is wider than the vector element type then it is implicitly
     296             :   // truncated.  Make that explicit here.
     297         316 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     298         316 :   SDValue InOp = N->getOperand(0);
     299         324 :   if (InOp.getValueType() != EltVT)
     300           0 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
     301         158 :   return InOp;
     302             : }
     303             : 
     304         153 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
     305         306 :   SDValue Cond = N->getOperand(0);
     306         306 :   EVT OpVT = Cond.getValueType();
     307         306 :   SDLoc DL(N);
     308             :   // The vselect result and true/value operands needs scalarizing, but it's
     309             :   // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
     310             :   // See the similar logic in ScalarizeVecRes_VSETCC
     311         306 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     312         137 :     Cond = GetScalarizedVector(Cond);
     313             :   } else {
     314          16 :     EVT VT = OpVT.getVectorElementType();
     315          32 :     Cond = DAG.getNode(
     316             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
     317          64 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     318             :   }
     319             : 
     320         306 :   SDValue LHS = GetScalarizedVector(N->getOperand(1));
     321             :   TargetLowering::BooleanContent ScalarBool =
     322         306 :       TLI.getBooleanContents(false, false);
     323         306 :   TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
     324             : 
     325             :   // If integer and float booleans have different contents then we can't
     326             :   // reliably optimize in all cases. There is a full explanation for this in
     327             :   // DAGCombiner::visitSELECT() where the same issue affects folding
     328             :   // (select C, 0, 1) to (xor C, 1).
     329         306 :   if (TLI.getBooleanContents(false, false) !=
     330         306 :       TLI.getBooleanContents(false, true)) {
     331             :     // At least try the common case where the boolean is generated by a
     332             :     // comparison.
     333           0 :     if (Cond->getOpcode() == ISD::SETCC) {
     334           0 :       EVT OpVT = Cond->getOperand(0)->getValueType(0);
     335           0 :       ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
     336           0 :       VecBool = TLI.getBooleanContents(OpVT);
     337             :     } else
     338             :       ScalarBool = TargetLowering::UndefinedBooleanContent;
     339             :   }
     340             : 
     341         153 :   if (ScalarBool != VecBool) {
     342          82 :     EVT CondVT = Cond.getValueType();
     343          41 :     switch (ScalarBool) {
     344             :       case TargetLowering::UndefinedBooleanContent:
     345             :         break;
     346          41 :       case TargetLowering::ZeroOrOneBooleanContent:
     347             :         assert(VecBool == TargetLowering::UndefinedBooleanContent ||
     348             :                VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
     349             :         // Vector read from all ones, scalar expects a single 1 so mask.
     350         123 :         Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
     351         123 :                            Cond, DAG.getConstant(1, SDLoc(N), CondVT));
     352          41 :         break;
     353           0 :       case TargetLowering::ZeroOrNegativeOneBooleanContent:
     354             :         assert(VecBool == TargetLowering::UndefinedBooleanContent ||
     355             :                VecBool == TargetLowering::ZeroOrOneBooleanContent);
     356             :         // Vector reads from a one, scalar from all ones so sign extend.
     357           0 :         Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
     358           0 :                            Cond, DAG.getValueType(MVT::i1));
     359           0 :         break;
     360             :     }
     361             :   }
     362             : 
     363         459 :   return DAG.getSelect(SDLoc(N),
     364             :                        LHS.getValueType(), Cond, LHS,
     365         765 :                        GetScalarizedVector(N->getOperand(2)));
     366             : }
     367             : 
     368         197 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
     369         394 :   SDValue LHS = GetScalarizedVector(N->getOperand(1));
     370         591 :   return DAG.getSelect(SDLoc(N),
     371         394 :                        LHS.getValueType(), N->getOperand(0), LHS,
     372        1182 :                        GetScalarizedVector(N->getOperand(2)));
     373             : }
     374             : 
     375           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
     376           0 :   SDValue LHS = GetScalarizedVector(N->getOperand(2));
     377           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
     378           0 :                      N->getOperand(0), N->getOperand(1),
     379           0 :                      LHS, GetScalarizedVector(N->getOperand(3)),
     380           0 :                      N->getOperand(4));
     381             : }
     382             : 
     383         333 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
     384             :   assert(N->getValueType(0).isVector() ==
     385             :          N->getOperand(0).getValueType().isVector() &&
     386             :          "Scalar/Vector type mismatch");
     387             : 
     388         999 :   if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N);
     389             : 
     390           0 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     391           0 :   SDValue RHS = GetScalarizedVector(N->getOperand(1));
     392           0 :   SDLoc DL(N);
     393             : 
     394             :   // Turn it into a scalar SETCC.
     395           0 :   return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2));
     396             : }
     397             : 
     398         137 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
     399         274 :   return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
     400             : }
     401             : 
     402           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
     403             :   // Figure out if the scalar is the LHS or RHS and return it.
     404           0 :   SDValue Arg = N->getOperand(2).getOperand(0);
     405           0 :   if (Arg.isUndef())
     406           0 :     return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
     407           0 :   unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
     408           0 :   return GetScalarizedVector(N->getOperand(Op));
     409             : }
     410             : 
     411         333 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
     412             :   assert(N->getValueType(0).isVector() &&
     413             :          N->getOperand(0).getValueType().isVector() &&
     414             :          "Operand types must be vectors");
     415         666 :   SDValue LHS = N->getOperand(0);
     416         666 :   SDValue RHS = N->getOperand(1);
     417         666 :   EVT OpVT = LHS.getValueType();
     418         666 :   EVT NVT = N->getValueType(0).getVectorElementType();
     419         666 :   SDLoc DL(N);
     420             : 
     421             :   // The result needs scalarizing, but it's not a given that the source does.
     422         666 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     423         327 :     LHS = GetScalarizedVector(LHS);
     424         327 :     RHS = GetScalarizedVector(RHS);
     425             :   } else {
     426           6 :     EVT VT = OpVT.getVectorElementType();
     427          12 :     LHS = DAG.getNode(
     428             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
     429          24 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     430          12 :     RHS = DAG.getNode(
     431             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
     432          24 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     433             :   }
     434             : 
     435             :   // Turn it into a scalar SETCC.
     436         333 :   SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
     437         999 :                             N->getOperand(2));
     438             :   // Vectors may have a different boolean contents to scalars.  Promote the
     439             :   // value appropriately.
     440             :   ISD::NodeType ExtendCode =
     441         666 :       TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
     442         999 :   return DAG.getNode(ExtendCode, DL, NVT, Res);
     443             : }
     444             : 
     445             : 
     446             : //===----------------------------------------------------------------------===//
     447             : //  Operand Vector Scalarization <1 x ty> -> ty.
     448             : //===----------------------------------------------------------------------===//
     449             : 
     450       16627 : bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
     451             :   DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
     452             :         N->dump(&DAG);
     453             :         dbgs() << "\n");
     454       16627 :   SDValue Res = SDValue();
     455             : 
     456       16627 :   if (!Res.getNode()) {
     457       33254 :     switch (N->getOpcode()) {
     458           0 :     default:
     459             : #ifndef NDEBUG
     460             :       dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
     461             :       N->dump(&DAG);
     462             :       dbgs() << "\n";
     463             : #endif
     464           0 :       llvm_unreachable("Do not know how to scalarize this operator's operand!");
     465        3327 :     case ISD::BITCAST:
     466        3327 :       Res = ScalarizeVecOp_BITCAST(N);
     467        3327 :       break;
     468          12 :     case ISD::ANY_EXTEND:
     469             :     case ISD::ZERO_EXTEND:
     470             :     case ISD::SIGN_EXTEND:
     471             :     case ISD::TRUNCATE:
     472             :     case ISD::FP_TO_SINT:
     473             :     case ISD::FP_TO_UINT:
     474             :     case ISD::SINT_TO_FP:
     475             :     case ISD::UINT_TO_FP:
     476          12 :       Res = ScalarizeVecOp_UnaryOp(N);
     477          12 :       break;
     478        5688 :     case ISD::CONCAT_VECTORS:
     479        5688 :       Res = ScalarizeVecOp_CONCAT_VECTORS(N);
     480        5688 :       break;
     481        4826 :     case ISD::EXTRACT_VECTOR_ELT:
     482        4826 :       Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
     483        4826 :       break;
     484           1 :     case ISD::VSELECT:
     485           1 :       Res = ScalarizeVecOp_VSELECT(N);
     486           1 :       break;
     487           8 :     case ISD::SETCC:
     488           8 :       Res = ScalarizeVecOp_VSETCC(N);
     489           8 :       break;
     490        2762 :     case ISD::STORE:
     491        2762 :       Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
     492        2762 :       break;
     493           3 :     case ISD::FP_ROUND:
     494           3 :       Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
     495           3 :       break;
     496             :     }
     497             :   }
     498             : 
     499             :   // If the result is null, the sub-method took care of registering results etc.
     500       16627 :   if (!Res.getNode()) return false;
     501             : 
     502             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
     503             :   // core about this.
     504       16627 :   if (Res.getNode() == N)
     505             :     return true;
     506             : 
     507             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
     508             :          "Invalid operand expansion");
     509             : 
     510       16627 :   ReplaceValueWith(SDValue(N, 0), Res);
     511       16627 :   return false;
     512             : }
     513             : 
     514             : /// If the value to convert is a vector that needs to be scalarized, it must be
     515             : /// <1 x ty>. Convert the element instead.
     516        3327 : SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
     517        6654 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     518        9981 :   return DAG.getNode(ISD::BITCAST, SDLoc(N),
     519       13308 :                      N->getValueType(0), Elt);
     520             : }
     521             : 
     522             : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
     523             : /// Do the operation on the element instead.
     524          12 : SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
     525             :   assert(N->getValueType(0).getVectorNumElements() == 1 &&
     526             :          "Unexpected vector type!");
     527          24 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     528          36 :   SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
     529          60 :                            N->getValueType(0).getScalarType(), Elt);
     530             :   // Revectorize the result so the types line up with what the uses of this
     531             :   // expression expect.
     532          60 :   return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Op);
     533             : }
     534             : 
     535             : /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
     536        5688 : SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
     537       17064 :   SmallVector<SDValue, 8> Ops(N->getNumOperands());
     538       22764 :   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
     539       34164 :     Ops[i] = GetScalarizedVector(N->getOperand(i));
     540       34128 :   return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
     541             : }
     542             : 
     543             : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
     544             : /// so just return the element, ignoring the index.
     545        4826 : SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
     546        9652 :   EVT VT = N->getValueType(0);
     547        9652 :   SDValue Res = GetScalarizedVector(N->getOperand(0));
     548        9652 :   if (Res.getValueType() != VT)
     549         900 :     Res = VT.isFloatingPoint()
     550        2253 :               ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
     551        2241 :               : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
     552        4826 :   return Res;
     553             : }
     554             : 
     555             : /// If the input condition is a vector that needs to be scalarized, it must be
     556             : /// <1 x i1>, so just convert to a normal ISD::SELECT
     557             : /// (still with vector output type since that was acceptable if we got here).
     558           1 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
     559           2 :   SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
     560           2 :   EVT VT = N->getValueType(0);
     561             : 
     562           5 :   return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
     563           4 :                      N->getOperand(2));
     564             : }
     565             : 
     566             : /// If the operand is a vector that needs to be scalarized then the
     567             : /// result must be v1i1, so just convert to a scalar SETCC and wrap
     568             : /// with a scalar_to_vector since the res type is legal if we got here
     569           8 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
     570             :   assert(N->getValueType(0).isVector() &&
     571             :          N->getOperand(0).getValueType().isVector() &&
     572             :          "Operand types must be vectors");
     573             :   assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
     574             : 
     575          16 :   EVT VT = N->getValueType(0);
     576          16 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     577          16 :   SDValue RHS = GetScalarizedVector(N->getOperand(1));
     578             : 
     579          24 :   EVT OpVT = N->getOperand(0).getValueType();
     580           8 :   EVT NVT = VT.getVectorElementType();
     581          16 :   SDLoc DL(N);
     582             :   // Turn it into a scalar SETCC.
     583           8 :   SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
     584          24 :       N->getOperand(2));
     585             : 
     586             :   // Vectors may have a different boolean contents to scalars.  Promote the
     587             :   // value appropriately.
     588             :   ISD::NodeType ExtendCode =
     589          16 :       TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
     590             : 
     591          16 :   Res = DAG.getNode(ExtendCode, DL, NVT, Res);
     592             : 
     593          24 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
     594             : }
     595             : 
     596             : /// If the value to store is a vector that needs to be scalarized, it must be
     597             : /// <1 x ty>. Just store the element.
     598        2762 : SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
     599             :   assert(N->isUnindexed() && "Indexed store of one-element vector?");
     600             :   assert(OpNo == 1 && "Do not know how to scalarize this operand!");
     601        5524 :   SDLoc dl(N);
     602             : 
     603        5524 :   if (N->isTruncatingStore())
     604           0 :     return DAG.getTruncStore(
     605           0 :         N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
     606           0 :         N->getBasePtr(), N->getPointerInfo(),
     607           0 :         N->getMemoryVT().getVectorElementType(), N->getAlignment(),
     608           0 :         N->getMemOperand()->getFlags(), N->getAAInfo());
     609             : 
     610       13810 :   return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
     611        8286 :                       N->getBasePtr(), N->getPointerInfo(),
     612        2762 :                       N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
     613       11048 :                       N->getAAInfo());
     614             : }
     615             : 
     616             : /// If the value to round is a vector that needs to be scalarized, it must be
     617             : /// <1 x ty>. Convert the element instead.
     618           3 : SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
     619           6 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     620           9 :   SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
     621           9 :                             N->getValueType(0).getVectorElementType(), Elt,
     622          15 :                             N->getOperand(1));
     623          15 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
     624             : }
     625             : 
     626             : //===----------------------------------------------------------------------===//
     627             : //  Result Vector Splitting
     628             : //===----------------------------------------------------------------------===//
     629             : 
     630             : /// This method is called when the specified result of the specified node is
     631             : /// found to need vector splitting. At this point, the node may also have
     632             : /// invalid operands or may have other results that need legalization, we just
     633             : /// know that (at least) one result needs vector splitting.
     634       63866 : void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
     635             :   DEBUG(dbgs() << "Split node result: ";
     636             :         N->dump(&DAG);
     637             :         dbgs() << "\n");
     638       63866 :   SDValue Lo, Hi;
     639             : 
     640             :   // See if the target wants to custom expand this node.
     641      127732 :   if (CustomLowerNode(N, N->getValueType(ResNo), true))
     642         285 :     return;
     643             : 
     644      127162 :   switch (N->getOpcode()) {
     645           0 :   default:
     646             : #ifndef NDEBUG
     647             :     dbgs() << "SplitVectorResult #" << ResNo << ": ";
     648             :     N->dump(&DAG);
     649             :     dbgs() << "\n";
     650             : #endif
     651           0 :     report_fatal_error("Do not know how to split the result of this "
     652             :                        "operator!\n");
     653             : 
     654           0 :   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
     655        1241 :   case ISD::VSELECT:
     656        1241 :   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
     657           0 :   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
     658         502 :   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
     659       17748 :   case ISD::BITCAST:           SplitVecRes_BITCAST(N, Lo, Hi); break;
     660        6710 :   case ISD::BUILD_VECTOR:      SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
     661       11549 :   case ISD::CONCAT_VECTORS:    SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
     662        3463 :   case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
     663           3 :   case ISD::INSERT_SUBVECTOR:  SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
     664           0 :   case ISD::FP_ROUND_INREG:    SplitVecRes_InregOp(N, Lo, Hi); break;
     665           8 :   case ISD::FPOWI:             SplitVecRes_FPOWI(N, Lo, Hi); break;
     666          17 :   case ISD::FCOPYSIGN:         SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
     667         647 :   case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
     668         106 :   case ISD::SCALAR_TO_VECTOR:  SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
     669          15 :   case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
     670        2872 :   case ISD::LOAD:
     671        2872 :     SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
     672        2872 :     break;
     673          18 :   case ISD::MLOAD:
     674          18 :     SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
     675          18 :     break;
     676          13 :   case ISD::MGATHER:
     677          13 :     SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
     678          13 :     break;
     679        1678 :   case ISD::SETCC:
     680        1678 :     SplitVecRes_SETCC(N, Lo, Hi);
     681        1678 :     break;
     682        1014 :   case ISD::VECTOR_SHUFFLE:
     683        1014 :     SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
     684        1014 :     break;
     685             : 
     686         497 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     687             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     688             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     689         497 :     SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
     690         497 :     break;
     691             : 
     692        2727 :   case ISD::BITREVERSE:
     693             :   case ISD::BSWAP:
     694             :   case ISD::CTLZ:
     695             :   case ISD::CTTZ:
     696             :   case ISD::CTLZ_ZERO_UNDEF:
     697             :   case ISD::CTTZ_ZERO_UNDEF:
     698             :   case ISD::CTPOP:
     699             :   case ISD::FABS:
     700             :   case ISD::FCEIL:
     701             :   case ISD::FCOS:
     702             :   case ISD::FEXP:
     703             :   case ISD::FEXP2:
     704             :   case ISD::FFLOOR:
     705             :   case ISD::FLOG:
     706             :   case ISD::FLOG10:
     707             :   case ISD::FLOG2:
     708             :   case ISD::FNEARBYINT:
     709             :   case ISD::FNEG:
     710             :   case ISD::FP_EXTEND:
     711             :   case ISD::FP_ROUND:
     712             :   case ISD::FP_TO_SINT:
     713             :   case ISD::FP_TO_UINT:
     714             :   case ISD::FRINT:
     715             :   case ISD::FROUND:
     716             :   case ISD::FSIN:
     717             :   case ISD::FSQRT:
     718             :   case ISD::FTRUNC:
     719             :   case ISD::SINT_TO_FP:
     720             :   case ISD::TRUNCATE:
     721             :   case ISD::UINT_TO_FP:
     722             :   case ISD::FCANONICALIZE:
     723        2727 :     SplitVecRes_UnaryOp(N, Lo, Hi);
     724        2727 :     break;
     725             : 
     726        7287 :   case ISD::ANY_EXTEND:
     727             :   case ISD::SIGN_EXTEND:
     728             :   case ISD::ZERO_EXTEND:
     729        7287 :     SplitVecRes_ExtendOp(N, Lo, Hi);
     730        7287 :     break;
     731             : 
     732        5296 :   case ISD::ADD:
     733             :   case ISD::SUB:
     734             :   case ISD::MUL:
     735             :   case ISD::MULHS:
     736             :   case ISD::MULHU:
     737             :   case ISD::FADD:
     738             :   case ISD::FSUB:
     739             :   case ISD::FMUL:
     740             :   case ISD::FMINNUM:
     741             :   case ISD::FMAXNUM:
     742             :   case ISD::FMINNAN:
     743             :   case ISD::FMAXNAN:
     744             :   case ISD::SDIV:
     745             :   case ISD::UDIV:
     746             :   case ISD::FDIV:
     747             :   case ISD::FPOW:
     748             :   case ISD::AND:
     749             :   case ISD::OR:
     750             :   case ISD::XOR:
     751             :   case ISD::SHL:
     752             :   case ISD::SRA:
     753             :   case ISD::SRL:
     754             :   case ISD::UREM:
     755             :   case ISD::SREM:
     756             :   case ISD::FREM:
     757             :   case ISD::SMIN:
     758             :   case ISD::SMAX:
     759             :   case ISD::UMIN:
     760             :   case ISD::UMAX:
     761        5296 :     SplitVecRes_BinOp(N, Lo, Hi);
     762        5296 :     break;
     763         170 :   case ISD::FMA:
     764         170 :     SplitVecRes_TernaryOp(N, Lo, Hi);
     765         170 :     break;
     766             :   }
     767             : 
     768             :   // If Lo/Hi is null, the sub-method took care of registering results etc.
     769       63581 :   if (Lo.getNode())
     770       63581 :     SetSplitVector(SDValue(N, ResNo), Lo, Hi);
     771             : }
     772             : 
     773        5296 : void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
     774             :                                          SDValue &Hi) {
     775        5296 :   SDValue LHSLo, LHSHi;
     776       10592 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
     777        5296 :   SDValue RHSLo, RHSHi;
     778       10592 :   GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
     779       10592 :   SDLoc dl(N);
     780             : 
     781       10592 :   const SDNodeFlags Flags = N->getFlags();
     782       10592 :   unsigned Opcode = N->getOpcode();
     783       10592 :   Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
     784       10592 :   Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
     785        5296 : }
     786             : 
     787         170 : void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
     788             :                                              SDValue &Hi) {
     789         170 :   SDValue Op0Lo, Op0Hi;
     790         340 :   GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
     791         170 :   SDValue Op1Lo, Op1Hi;
     792         340 :   GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
     793         170 :   SDValue Op2Lo, Op2Hi;
     794         340 :   GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
     795         340 :   SDLoc dl(N);
     796             : 
     797         340 :   Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
     798         340 :                    Op0Lo, Op1Lo, Op2Lo);
     799         340 :   Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
     800         340 :                    Op0Hi, Op1Hi, Op2Hi);
     801         170 : }
     802             : 
     803       17748 : void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
     804             :                                            SDValue &Hi) {
     805             :   // We know the result is a vector.  The input may be either a vector or a
     806             :   // scalar value.
     807       17748 :   EVT LoVT, HiVT;
     808       70992 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     809       31366 :   SDLoc dl(N);
     810             : 
     811       35496 :   SDValue InOp = N->getOperand(0);
     812       35496 :   EVT InVT = InOp.getValueType();
     813             : 
     814             :   // Handle some special cases efficiently.
     815       35496 :   switch (getTypeAction(InVT)) {
     816             :   case TargetLowering::TypeLegal:
     817             :   case TargetLowering::TypePromoteInteger:
     818             :   case TargetLowering::TypePromoteFloat:
     819             :   case TargetLowering::TypeSoftenFloat:
     820             :   case TargetLowering::TypeScalarizeVector:
     821             :   case TargetLowering::TypeWidenVector:
     822             :     break;
     823        1633 :   case TargetLowering::TypeExpandInteger:
     824             :   case TargetLowering::TypeExpandFloat:
     825             :     // A scalar to vector conversion, where the scalar needs expansion.
     826             :     // If the vector is being split in two then we can just convert the
     827             :     // expanded pieces.
     828        1633 :     if (LoVT == HiVT) {
     829        1633 :       GetExpandedOp(InOp, Lo, Hi);
     830        3266 :       if (DAG.getDataLayout().isBigEndian())
     831             :         std::swap(Lo, Hi);
     832        3266 :       Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     833        3266 :       Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     834        5763 :       return;
     835             :     }
     836             :     break;
     837        2497 :   case TargetLowering::TypeSplitVector:
     838             :     // If the input is a vector that needs to be split, convert each split
     839             :     // piece of the input now.
     840        2497 :     GetSplitVector(InOp, Lo, Hi);
     841        4994 :     Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     842        4994 :     Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     843        2497 :     return;
     844             :   }
     845             : 
     846             :   // In the general case, convert the input to an integer and split it by hand.
     847       13618 :   EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
     848       13618 :   EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
     849       27236 :   if (DAG.getDataLayout().isBigEndian())
     850             :     std::swap(LoIntVT, HiIntVT);
     851             : 
     852       13618 :   SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
     853             : 
     854       27236 :   if (DAG.getDataLayout().isBigEndian())
     855             :     std::swap(Lo, Hi);
     856       27236 :   Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     857       27236 :   Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     858             : }
     859             : 
     860        6710 : void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
     861             :                                                 SDValue &Hi) {
     862        6710 :   EVT LoVT, HiVT;
     863       13420 :   SDLoc dl(N);
     864       26840 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     865        6710 :   unsigned LoNumElts = LoVT.getVectorNumElements();
     866       20130 :   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
     867       13420 :   Lo = DAG.getBuildVector(LoVT, dl, LoOps);
     868             : 
     869       26840 :   SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
     870       13420 :   Hi = DAG.getBuildVector(HiVT, dl, HiOps);
     871        6710 : }
     872             : 
     873       11549 : void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
     874             :                                                   SDValue &Hi) {
     875             :   assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
     876       14724 :   SDLoc dl(N);
     877       23098 :   unsigned NumSubvectors = N->getNumOperands() / 2;
     878       11549 :   if (NumSubvectors == 1) {
     879       16748 :     Lo = N->getOperand(0);
     880       16748 :     Hi = N->getOperand(1);
     881        8374 :     return;
     882             :   }
     883             : 
     884             :   EVT LoVT, HiVT;
     885       12700 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     886             : 
     887        9525 :   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
     888        9525 :   Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
     889             : 
     890       12700 :   SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
     891        9525 :   Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
     892             : }
     893             : 
     894        3463 : void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
     895             :                                                      SDValue &Hi) {
     896        6926 :   SDValue Vec = N->getOperand(0);
     897        6926 :   SDValue Idx = N->getOperand(1);
     898        6926 :   SDLoc dl(N);
     899             : 
     900        3463 :   EVT LoVT, HiVT;
     901       13852 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     902             : 
     903        6926 :   Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
     904        6926 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
     905        6926 :   Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
     906        3463 :                    DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
     907       17315 :                                    TLI.getVectorIdxTy(DAG.getDataLayout())));
     908        3463 : }
     909             : 
     910           3 : void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
     911             :                                                     SDValue &Hi) {
     912           6 :   SDValue Vec = N->getOperand(0);
     913           6 :   SDValue SubVec = N->getOperand(1);
     914           6 :   SDValue Idx = N->getOperand(2);
     915           3 :   SDLoc dl(N);
     916           3 :   GetSplitVector(Vec, Lo, Hi);
     917             : 
     918           6 :   EVT VecVT = Vec.getValueType();
     919           3 :   unsigned VecElems = VecVT.getVectorNumElements();
     920           6 :   unsigned SubElems = SubVec.getValueType().getVectorNumElements();
     921             : 
     922             :   // If we know the index is 0, and we know the subvector doesn't cross the
     923             :   // boundary between the halves, we can avoid spilling the vector, and insert
     924             :   // into the lower half of the split vector directly.
     925             :   // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
     926             :   // the index is constant and there is no boundary crossing. But those cases
     927             :   // don't seem to get hit in practice.
     928           3 :   if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
     929           3 :     unsigned IdxVal = ConstIdx->getZExtValue();
     930           3 :     if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
     931             :       EVT LoVT, HiVT;
     932          12 :       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     933           3 :       Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
     934             :       return;
     935             :     }
     936             :   }
     937             : 
     938             :   // Spill the vector to the stack.
     939           0 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
     940             :   SDValue Store =
     941           0 :       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
     942             : 
     943             :   // Store the new subvector into the specified index.
     944           0 :   SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
     945           0 :   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
     946           0 :   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
     947           0 :   Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
     948             : 
     949             :   // Load the Lo part from the stack slot.
     950           0 :   Lo =
     951           0 :       DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
     952             : 
     953             :   // Increment the pointer to the other part.
     954           0 :   unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
     955           0 :   StackPtr =
     956           0 :       DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
     957           0 :                   DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
     958             : 
     959             :   // Load the Hi part from the stack slot.
     960           0 :   Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
     961           0 :                    MinAlign(Alignment, IncrementSize));
     962             : }
     963             : 
     964           8 : void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
     965             :                                          SDValue &Hi) {
     966          16 :   SDLoc dl(N);
     967          16 :   GetSplitVector(N->getOperand(0), Lo, Hi);
     968          32 :   Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
     969          32 :   Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
     970           8 : }
     971             : 
     972          17 : void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
     973             :                                              SDValue &Hi) {
     974          17 :   SDValue LHSLo, LHSHi;
     975          34 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
     976          34 :   SDLoc DL(N);
     977             : 
     978          17 :   SDValue RHSLo, RHSHi;
     979          34 :   SDValue RHS = N->getOperand(1);
     980          34 :   EVT RHSVT = RHS.getValueType();
     981          34 :   if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
     982          15 :     GetSplitVector(RHS, RHSLo, RHSHi);
     983             :   else
     984          10 :     std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
     985             : 
     986             : 
     987          51 :   Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
     988          51 :   Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
     989          17 : }
     990             : 
     991          15 : void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
     992             :                                            SDValue &Hi) {
     993          15 :   SDValue LHSLo, LHSHi;
     994          30 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
     995          30 :   SDLoc dl(N);
     996             : 
     997             :   EVT LoVT, HiVT;
     998          15 :   std::tie(LoVT, HiVT) =
     999          75 :     DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
    1000             : 
    1001          30 :   Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
    1002          45 :                    DAG.getValueType(LoVT));
    1003          30 :   Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
    1004          45 :                    DAG.getValueType(HiVT));
    1005          15 : }
    1006             : 
    1007         503 : void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
    1008             :                                                  SDValue &Hi) {
    1009        1006 :   unsigned Opcode = N->getOpcode();
    1010        1006 :   SDValue N0 = N->getOperand(0);
    1011             : 
    1012        1006 :   SDLoc dl(N);
    1013         503 :   SDValue InLo, InHi;
    1014             : 
    1015        1509 :   if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
    1016         501 :     GetSplitVector(N0, InLo, InHi);
    1017             :   else
    1018           6 :     std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
    1019             : 
    1020        1006 :   EVT InLoVT = InLo.getValueType();
    1021         503 :   unsigned InNumElements = InLoVT.getVectorNumElements();
    1022             : 
    1023         503 :   EVT OutLoVT, OutHiVT;
    1024        2012 :   std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1025         503 :   unsigned OutNumElements = OutLoVT.getVectorNumElements();
    1026             :   assert((2 * OutNumElements) <= InNumElements &&
    1027             :          "Illegal extend vector in reg split");
    1028             : 
    1029             :   // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
    1030             :   // input vector (i.e. we only use InLo):
    1031             :   // OutLo will extend the first OutNumElements from InLo.
    1032             :   // OutHi will extend the next OutNumElements from InLo.
    1033             : 
    1034             :   // Shuffle the elements from InLo for OutHi into the bottom elements to
    1035             :   // create a 'fake' InHi.
    1036        1509 :   SmallVector<int, 8> SplitHi(InNumElements, -1);
    1037        3565 :   for (unsigned i = 0; i != OutNumElements; ++i)
    1038        6124 :     SplitHi[i] = i + OutNumElements;
    1039        1006 :   InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
    1040             : 
    1041        1006 :   Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
    1042        1006 :   Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
    1043         503 : }
    1044             : 
    1045         647 : void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
    1046             :                                                      SDValue &Hi) {
    1047        1294 :   SDValue Vec = N->getOperand(0);
    1048        1294 :   SDValue Elt = N->getOperand(1);
    1049        1294 :   SDValue Idx = N->getOperand(2);
    1050         680 :   SDLoc dl(N);
    1051         647 :   GetSplitVector(Vec, Lo, Hi);
    1052             : 
    1053         614 :   if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
    1054         614 :     unsigned IdxVal = CIdx->getZExtValue();
    1055        1228 :     unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
    1056         614 :     if (IdxVal < LoNumElts)
    1057         700 :       Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
    1058         350 :                        Lo.getValueType(), Lo, Elt, Idx);
    1059             :     else
    1060         264 :       Hi =
    1061         792 :           DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
    1062         264 :                       DAG.getConstant(IdxVal - LoNumElts, dl,
    1063        1320 :                                       TLI.getVectorIdxTy(DAG.getDataLayout())));
    1064         614 :     return;
    1065             :   }
    1066             : 
    1067             :   // See if the target wants to custom expand this node.
    1068          66 :   if (CustomLowerNode(N, N->getValueType(0), true))
    1069             :     return;
    1070             : 
    1071             :   // Spill the vector to the stack.
    1072          66 :   EVT VecVT = Vec.getValueType();
    1073          33 :   EVT EltVT = VecVT.getVectorElementType();
    1074          33 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1075             :   SDValue Store =
    1076         132 :       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
    1077             : 
    1078             :   // Store the new element.  This may be larger than the vector element type,
    1079             :   // so use a truncating store.
    1080          33 :   SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1081          33 :   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
    1082          66 :   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
    1083          33 :   Store =
    1084         132 :       DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT);
    1085             : 
    1086             :   // Load the Lo part from the stack slot.
    1087          33 :   Lo =
    1088         165 :       DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
    1089             : 
    1090             :   // Increment the pointer to the other part.
    1091          33 :   unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
    1092          66 :   StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
    1093             :                          DAG.getConstant(IncrementSize, dl,
    1094          99 :                                          StackPtr.getValueType()));
    1095             : 
    1096             :   // Load the Hi part from the stack slot.
    1097          66 :   Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
    1098         132 :                    MinAlign(Alignment, IncrementSize));
    1099             : }
    1100             : 
    1101         106 : void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
    1102             :                                                     SDValue &Hi) {
    1103         106 :   EVT LoVT, HiVT;
    1104         212 :   SDLoc dl(N);
    1105         424 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1106         318 :   Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
    1107         106 :   Hi = DAG.getUNDEF(HiVT);
    1108         106 : }
    1109             : 
    1110        2872 : void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
    1111             :                                         SDValue &Hi) {
    1112             :   assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
    1113        2872 :   EVT LoVT, HiVT;
    1114        5744 :   SDLoc dl(LD);
    1115       11488 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
    1116             : 
    1117        2872 :   ISD::LoadExtType ExtType = LD->getExtensionType();
    1118        5744 :   SDValue Ch = LD->getChain();
    1119        2872 :   SDValue Ptr = LD->getBasePtr();
    1120        5744 :   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
    1121        2872 :   EVT MemoryVT = LD->getMemoryVT();
    1122        5744 :   unsigned Alignment = LD->getOriginalAlignment();
    1123        2872 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    1124        2872 :   AAMDNodes AAInfo = LD->getAAInfo();
    1125             : 
    1126        2872 :   EVT LoMemVT, HiMemVT;
    1127        8616 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1128             : 
    1129        5744 :   Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
    1130        2872 :                    LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
    1131             : 
    1132        2872 :   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
    1133        5744 :   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
    1134        8616 :                     DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
    1135        5744 :   Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
    1136        2872 :                    LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
    1137        5744 :                    Alignment, MMOFlags, AAInfo);
    1138             : 
    1139             :   // Build a factor node to remember that this load is independent of the
    1140             :   // other one.
    1141        5744 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1142       14360 :                    Hi.getValue(1));
    1143             : 
    1144             :   // Legalize the chain result - switch anything that used the old chain to
    1145             :   // use the new one.
    1146        2872 :   ReplaceValueWith(SDValue(LD, 1), Ch);
    1147        2872 : }
    1148             : 
    1149          18 : void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
    1150             :                                          SDValue &Lo, SDValue &Hi) {
    1151          18 :   EVT LoVT, HiVT;
    1152          36 :   SDLoc dl(MLD);
    1153          72 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
    1154             : 
    1155          36 :   SDValue Ch = MLD->getChain();
    1156          36 :   SDValue Ptr = MLD->getBasePtr();
    1157          36 :   SDValue Mask = MLD->getMask();
    1158          18 :   SDValue Src0 = MLD->getSrc0();
    1159          36 :   unsigned Alignment = MLD->getOriginalAlignment();
    1160          18 :   ISD::LoadExtType ExtType = MLD->getExtensionType();
    1161             : 
    1162             :   // if Alignment is equal to the vector size,
    1163             :   // take the half of it for the second part
    1164             :   unsigned SecondHalfAlignment =
    1165          54 :     (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
    1166          18 :      Alignment/2 : Alignment;
    1167             : 
    1168             :   // Split Mask operand
    1169          18 :   SDValue MaskLo, MaskHi;
    1170          54 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1171           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1172             :   else
    1173          54 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1174             : 
    1175          18 :   EVT MemoryVT = MLD->getMemoryVT();
    1176          18 :   EVT LoMemVT, HiMemVT;
    1177          54 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1178             : 
    1179          18 :   SDValue Src0Lo, Src0Hi;
    1180          54 :   if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
    1181          18 :     GetSplitVector(Src0, Src0Lo, Src0Hi);
    1182             :   else
    1183           0 :     std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
    1184             : 
    1185          18 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1186          36 :     getMachineMemOperand(MLD->getPointerInfo(),
    1187          18 :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1188          54 :                          Alignment, MLD->getAAInfo(), MLD->getRanges());
    1189             : 
    1190          36 :   Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
    1191          36 :                          ExtType, MLD->isExpandingLoad());
    1192             : 
    1193          36 :   Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
    1194          36 :                                    MLD->isExpandingLoad());
    1195             : 
    1196          36 :   MMO = DAG.getMachineFunction().
    1197          36 :     getMachineMemOperand(MLD->getPointerInfo(),
    1198          18 :                          MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),
    1199          54 :                          SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());
    1200             : 
    1201          36 :   Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
    1202          36 :                          ExtType, MLD->isExpandingLoad());
    1203             : 
    1204             : 
    1205             :   // Build a factor node to remember that this load is independent of the
    1206             :   // other one.
    1207          36 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1208          90 :                    Hi.getValue(1));
    1209             : 
    1210             :   // Legalize the chain result - switch anything that used the old chain to
    1211             :   // use the new one.
    1212          18 :   ReplaceValueWith(SDValue(MLD, 1), Ch);
    1213             : 
    1214          18 : }
    1215             : 
    1216          13 : void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
    1217             :                                          SDValue &Lo, SDValue &Hi) {
    1218          13 :   EVT LoVT, HiVT;
    1219          26 :   SDLoc dl(MGT);
    1220          52 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
    1221             : 
    1222          26 :   SDValue Ch = MGT->getChain();
    1223          26 :   SDValue Ptr = MGT->getBasePtr();
    1224          26 :   SDValue Mask = MGT->getMask();
    1225          26 :   SDValue Src0 = MGT->getValue();
    1226          26 :   SDValue Index = MGT->getIndex();
    1227          26 :   unsigned Alignment = MGT->getOriginalAlignment();
    1228             : 
    1229             :   // Split Mask operand
    1230          13 :   SDValue MaskLo, MaskHi;
    1231          39 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1232           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1233             :   else
    1234          39 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1235             : 
    1236          13 :   EVT MemoryVT = MGT->getMemoryVT();
    1237          13 :   EVT LoMemVT, HiMemVT;
    1238             :   // Split MemoryVT
    1239          39 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1240             : 
    1241          13 :   SDValue Src0Lo, Src0Hi;
    1242          39 :   if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
    1243          13 :     GetSplitVector(Src0, Src0Lo, Src0Hi);
    1244             :   else
    1245           0 :     std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
    1246             : 
    1247          13 :   SDValue IndexHi, IndexLo;
    1248          39 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1249           9 :     GetSplitVector(Index, IndexLo, IndexHi);
    1250             :   else
    1251          12 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
    1252             : 
    1253          13 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1254          26 :     getMachineMemOperand(MGT->getPointerInfo(),
    1255          13 :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1256          39 :                          Alignment, MGT->getAAInfo(), MGT->getRanges());
    1257             : 
    1258          13 :   SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
    1259          52 :   Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
    1260          39 :                            MMO);
    1261             : 
    1262          13 :   SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
    1263          52 :   Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
    1264          39 :                            MMO);
    1265             : 
    1266             :   // Build a factor node to remember that this load is independent of the
    1267             :   // other one.
    1268          26 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1269          65 :                    Hi.getValue(1));
    1270             : 
    1271             :   // Legalize the chain result - switch anything that used the old chain to
    1272             :   // use the new one.
    1273          13 :   ReplaceValueWith(SDValue(MGT, 1), Ch);
    1274          13 : }
    1275             : 
    1276             : 
    1277        1678 : void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
    1278             :   assert(N->getValueType(0).isVector() &&
    1279             :          N->getOperand(0).getValueType().isVector() &&
    1280             :          "Operand types must be vectors");
    1281             : 
    1282        1678 :   EVT LoVT, HiVT;
    1283        3356 :   SDLoc DL(N);
    1284        6712 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1285             : 
    1286             :   // Split the input.
    1287             :   SDValue LL, LH, RL, RH;
    1288        5034 :   std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
    1289        5034 :   std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
    1290             : 
    1291        5034 :   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
    1292        5034 :   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
    1293        1678 : }
    1294             : 
    1295       10011 : void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
    1296             :                                            SDValue &Hi) {
    1297             :   // Get the dest types - they may not match the input types, e.g. int_to_fp.
    1298       10011 :   EVT LoVT, HiVT;
    1299       20022 :   SDLoc dl(N);
    1300       40044 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1301             : 
    1302             :   // If the input also splits, handle it directly for a compile time speedup.
    1303             :   // Otherwise split it by hand.
    1304       30033 :   EVT InVT = N->getOperand(0).getValueType();
    1305       20022 :   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
    1306       12540 :     GetSplitVector(N->getOperand(0), Lo, Hi);
    1307             :   else
    1308       11223 :     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
    1309             : 
    1310       20022 :   if (N->getOpcode() == ISD::FP_ROUND) {
    1311        1539 :     Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
    1312        2052 :     Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
    1313             :   } else {
    1314       18996 :     Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
    1315       28494 :     Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
    1316             :   }
    1317       10011 : }
    1318             : 
    1319        7287 : void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
    1320             :                                             SDValue &Hi) {
    1321       14571 :   SDLoc dl(N);
    1322       21861 :   EVT SrcVT = N->getOperand(0).getValueType();
    1323       14574 :   EVT DestVT = N->getValueType(0);
    1324             :   EVT LoVT, HiVT;
    1325       21861 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
    1326             : 
    1327             :   // We can do better than a generic split operation if the extend is doing
    1328             :   // more than just doubling the width of the elements and the following are
    1329             :   // true:
    1330             :   //   - The number of vector elements is even,
    1331             :   //   - the source type is legal,
    1332             :   //   - the type of a split source is illegal,
    1333             :   //   - the type of an extended (by doubling element size) source is legal, and
    1334             :   //   - the type of that extended source when split is legal.
    1335             :   //
    1336             :   // This won't necessarily completely legalize the operation, but it will
    1337             :   // more effectively move in the right direction and prevent falling down
    1338             :   // to scalarization in many cases due to the input vector being split too
    1339             :   // far.
    1340        7287 :   unsigned NumElements = SrcVT.getVectorNumElements();
    1341       14574 :   if ((NumElements & 1) == 0 &&
    1342        7287 :       SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
    1343        4697 :     LLVMContext &Ctx = *DAG.getContext();
    1344        4697 :     EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
    1345        4697 :     EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
    1346             : 
    1347             :     EVT SplitLoVT, SplitHiVT;
    1348       14091 :     std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
    1349        4768 :     if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
    1350          73 :         TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
    1351             :       DEBUG(dbgs() << "Split vector extend via incremental extend:";
    1352             :             N->dump(&DAG); dbgs() << "\n");
    1353             :       // Extend the source vector by one step.
    1354             :       SDValue NewSrc =
    1355          12 :           DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
    1356             :       // Get the low and high halves of the new, extended one step, vector.
    1357           9 :       std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
    1358             :       // Extend those vector halves the rest of the way.
    1359           9 :       Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
    1360           9 :       Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
    1361             :       return;
    1362             :     }
    1363             :   }
    1364             :   // Fall back to the generic unary operator splitting otherwise.
    1365        7284 :   SplitVecRes_UnaryOp(N, Lo, Hi);
    1366             : }
    1367             : 
    1368        1014 : void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
    1369             :                                                   SDValue &Lo, SDValue &Hi) {
    1370             :   // The low and high parts of the original input give four input vectors.
    1371        1014 :   SDValue Inputs[4];
    1372        2028 :   SDLoc dl(N);
    1373        2028 :   GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
    1374        2028 :   GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
    1375        2028 :   EVT NewVT = Inputs[0].getValueType();
    1376        1014 :   unsigned NewElts = NewVT.getVectorNumElements();
    1377             : 
    1378             :   // If Lo or Hi uses elements from at most two of the four input vectors, then
    1379             :   // express it as a vector shuffle of those two inputs.  Otherwise extract the
    1380             :   // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
    1381        2028 :   SmallVector<int, 16> Ops;
    1382        3042 :   for (unsigned High = 0; High < 2; ++High) {
    1383        2028 :     SDValue &Output = High ? Hi : Lo;
    1384             : 
    1385             :     // Build a shuffle mask for the output, discovering on the fly which
    1386             :     // input vectors to use as shuffle operands (recorded in InputUsed).
    1387             :     // If building a suitable shuffle vector proves too hard, then bail
    1388             :     // out with useBuildVector set.
    1389        2028 :     unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
    1390        2028 :     unsigned FirstMaskIdx = High * NewElts;
    1391        2028 :     bool useBuildVector = false;
    1392       24379 :     for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
    1393             :       // The mask element.  This indexes into the input.
    1394       44880 :       int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
    1395             : 
    1396             :       // The input vector this mask element indexes into.
    1397       22440 :       unsigned Input = (unsigned)Idx / NewElts;
    1398             : 
    1399       32120 :       if (Input >= array_lengthof(Inputs)) {
    1400             :         // The mask element does not index into any input vector.
    1401        9680 :         Ops.push_back(-1);
    1402        9680 :         continue;
    1403             :       }
    1404             : 
    1405             :       // Turn the index into an offset from the start of the input vector.
    1406       12760 :       Idx -= Input * NewElts;
    1407             : 
    1408             :       // Find or create a shuffle vector operand to hold this input.
    1409             :       unsigned OpNo;
    1410       16300 :       for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
    1411       16211 :         if (InputUsed[OpNo] == Input) {
    1412             :           // This input vector is already an operand.
    1413             :           break;
    1414        5903 :         } else if (InputUsed[OpNo] == -1U) {
    1415             :           // Create a new operand for this input vector.
    1416        2363 :           InputUsed[OpNo] = Input;
    1417        2363 :           break;
    1418             :         }
    1419             :       }
    1420             : 
    1421       12760 :       if (OpNo >= array_lengthof(InputUsed)) {
    1422             :         // More than two input vectors used!  Give up on trying to create a
    1423             :         // shuffle vector.  Insert all elements into a BUILD_VECTOR instead.
    1424             :         useBuildVector = true;
    1425             :         break;
    1426             :       }
    1427             : 
    1428             :       // Add the mask index for the new shuffle vector.
    1429       12671 :       Ops.push_back(Idx + OpNo * NewElts);
    1430             :     }
    1431             : 
    1432        2028 :     if (useBuildVector) {
    1433          89 :       EVT EltVT = NewVT.getVectorElementType();
    1434         178 :       SmallVector<SDValue, 16> SVOps;
    1435             : 
    1436             :       // Extract the input elements by hand.
    1437        1441 :       for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
    1438             :         // The mask element.  This indexes into the input.
    1439        2704 :         int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
    1440             : 
    1441             :         // The input vector this mask element indexes into.
    1442        1352 :         unsigned Input = (unsigned)Idx / NewElts;
    1443             : 
    1444        1476 :         if (Input >= array_lengthof(Inputs)) {
    1445             :           // The mask element is "undef" or indexes off the end of the input.
    1446         124 :           SVOps.push_back(DAG.getUNDEF(EltVT));
    1447         124 :           continue;
    1448             :         }
    1449             : 
    1450             :         // Turn the index into an offset from the start of the input vector.
    1451        1228 :         Idx -= Input * NewElts;
    1452             : 
    1453             :         // Extract the vector element by hand.
    1454        2456 :         SVOps.push_back(DAG.getNode(
    1455             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
    1456        4912 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    1457             :       }
    1458             : 
    1459             :       // Construct the Lo/Hi output using a BUILD_VECTOR.
    1460         178 :       Output = DAG.getBuildVector(NewVT, dl, SVOps);
    1461        1939 :     } else if (InputUsed[0] == -1U) {
    1462             :       // No input vectors were used!  The result is undefined.
    1463         426 :       Output = DAG.getUNDEF(NewVT);
    1464             :     } else {
    1465        1513 :       SDValue Op0 = Inputs[InputUsed[0]];
    1466             :       // If only one input was used, use an undefined vector for the other.
    1467        1513 :       SDValue Op1 = InputUsed[1] == -1U ?
    1468        1513 :         DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
    1469             :       // At least one input vector was used.  Create a new shuffle vector.
    1470        3026 :       Output =  DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
    1471             :     }
    1472             : 
    1473        2028 :     Ops.clear();
    1474             :   }
    1475        1014 : }
    1476             : 
    1477             : 
    1478             : //===----------------------------------------------------------------------===//
    1479             : //  Operand Vector Splitting
    1480             : //===----------------------------------------------------------------------===//
    1481             : 
    1482             : /// This method is called when the specified operand of the specified node is
    1483             : /// found to need vector splitting. At this point, all of the result types of
    1484             : /// the node are known to be legal, but other operands of the node may need
    1485             : /// legalization as well as the specified one.
    1486       60853 : bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
    1487             :   DEBUG(dbgs() << "Split node operand: ";
    1488             :         N->dump(&DAG);
    1489             :         dbgs() << "\n");
    1490       60853 :   SDValue Res = SDValue();
    1491             : 
    1492             :   // See if the target wants to custom split this node.
    1493      182559 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    1494             :     return false;
    1495             : 
    1496       60454 :   if (!Res.getNode()) {
    1497      120908 :     switch (N->getOpcode()) {
    1498           0 :     default:
    1499             : #ifndef NDEBUG
    1500             :       dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
    1501             :       N->dump(&DAG);
    1502             :       dbgs() << "\n";
    1503             : #endif
    1504           0 :       report_fatal_error("Do not know how to split this operator's "
    1505             :                          "operand!\n");
    1506             : 
    1507        1046 :     case ISD::SETCC:             Res = SplitVecOp_VSETCC(N); break;
    1508        5261 :     case ISD::BITCAST:           Res = SplitVecOp_BITCAST(N); break;
    1509       24547 :     case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
    1510       13257 :     case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
    1511           0 :     case ISD::CONCAT_VECTORS:    Res = SplitVecOp_CONCAT_VECTORS(N); break;
    1512         663 :     case ISD::TRUNCATE:
    1513         663 :       Res = SplitVecOp_TruncateHelper(N);
    1514         663 :       break;
    1515          73 :     case ISD::FP_ROUND:          Res = SplitVecOp_FP_ROUND(N); break;
    1516           8 :     case ISD::FCOPYSIGN:         Res = SplitVecOp_FCOPYSIGN(N); break;
    1517        6847 :     case ISD::STORE:
    1518        6847 :       Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
    1519        6847 :       break;
    1520           6 :     case ISD::MSTORE:
    1521           6 :       Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
    1522           6 :       break;
    1523          17 :     case ISD::MSCATTER:
    1524          17 :       Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
    1525          17 :       break;
    1526          21 :     case ISD::MGATHER:
    1527          21 :       Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
    1528          21 :       break;
    1529          45 :     case ISD::VSELECT:
    1530          45 :       Res = SplitVecOp_VSELECT(N, OpNo);
    1531          45 :       break;
    1532          54 :     case ISD::FP_TO_SINT:
    1533             :     case ISD::FP_TO_UINT:
    1534         216 :       if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
    1535          38 :         Res = SplitVecOp_TruncateHelper(N);
    1536             :       else
    1537          16 :         Res = SplitVecOp_UnaryOp(N);
    1538             :       break;
    1539         209 :     case ISD::SINT_TO_FP:
    1540             :     case ISD::UINT_TO_FP:
    1541         836 :       if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
    1542         103 :         Res = SplitVecOp_TruncateHelper(N);
    1543             :       else
    1544         106 :         Res = SplitVecOp_UnaryOp(N);
    1545             :       break;
    1546        8381 :     case ISD::CTTZ:
    1547             :     case ISD::CTLZ:
    1548             :     case ISD::CTPOP:
    1549             :     case ISD::FP_EXTEND:
    1550             :     case ISD::SIGN_EXTEND:
    1551             :     case ISD::ZERO_EXTEND:
    1552             :     case ISD::ANY_EXTEND:
    1553             :     case ISD::FTRUNC:
    1554             :     case ISD::FCANONICALIZE:
    1555        8381 :       Res = SplitVecOp_UnaryOp(N);
    1556        8381 :       break;
    1557             : 
    1558           6 :     case ISD::ANY_EXTEND_VECTOR_INREG:
    1559             :     case ISD::SIGN_EXTEND_VECTOR_INREG:
    1560             :     case ISD::ZERO_EXTEND_VECTOR_INREG:
    1561           6 :       Res = SplitVecOp_ExtVecInRegOp(N);
    1562           6 :       break;
    1563             : 
    1564          13 :     case ISD::VECREDUCE_FADD:
    1565             :     case ISD::VECREDUCE_FMUL:
    1566             :     case ISD::VECREDUCE_ADD:
    1567             :     case ISD::VECREDUCE_MUL:
    1568             :     case ISD::VECREDUCE_AND:
    1569             :     case ISD::VECREDUCE_OR:
    1570             :     case ISD::VECREDUCE_XOR:
    1571             :     case ISD::VECREDUCE_SMAX:
    1572             :     case ISD::VECREDUCE_SMIN:
    1573             :     case ISD::VECREDUCE_UMAX:
    1574             :     case ISD::VECREDUCE_UMIN:
    1575             :     case ISD::VECREDUCE_FMAX:
    1576             :     case ISD::VECREDUCE_FMIN:
    1577          13 :       Res = SplitVecOp_VECREDUCE(N, OpNo);
    1578          13 :       break;
    1579             :     }
    1580             :   }
    1581             : 
    1582             :   // If the result is null, the sub-method took care of registering results etc.
    1583       60454 :   if (!Res.getNode()) return false;
    1584             : 
    1585             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    1586             :   // core about this.
    1587       60433 :   if (Res.getNode() == N)
    1588             :     return true;
    1589             : 
    1590             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    1591             :          "Invalid operand expansion");
    1592             : 
    1593       47980 :   ReplaceValueWith(SDValue(N, 0), Res);
    1594       47980 :   return false;
    1595             : }
    1596             : 
    1597          49 : SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
    1598             :   // The only possibility for an illegal operand is the mask, since result type
    1599             :   // legalization would have handled this node already otherwise.
    1600             :   assert(OpNo == 0 && "Illegal operand must be mask");
    1601             : 
    1602          98 :   SDValue Mask = N->getOperand(0);
    1603          98 :   SDValue Src0 = N->getOperand(1);
    1604          98 :   SDValue Src1 = N->getOperand(2);
    1605          98 :   EVT Src0VT = Src0.getValueType();
    1606          98 :   SDLoc DL(N);
    1607             :   assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
    1608             : 
    1609          49 :   SDValue Lo, Hi;
    1610          98 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1611             :   assert(Lo.getValueType() == Hi.getValueType() &&
    1612             :          "Lo and Hi have differing types");
    1613             : 
    1614             :   EVT LoOpVT, HiOpVT;
    1615         147 :   std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
    1616             :   assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
    1617             : 
    1618             :   SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
    1619         147 :   std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
    1620         147 :   std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
    1621         147 :   std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
    1622             : 
    1623             :   SDValue LoSelect =
    1624          49 :     DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
    1625             :   SDValue HiSelect =
    1626          49 :     DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
    1627             : 
    1628         147 :   return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
    1629             : }
    1630             : 
    1631          13 : SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
    1632          26 :   EVT ResVT = N->getValueType(0);
    1633          13 :   SDValue Lo, Hi;
    1634          26 :   SDLoc dl(N);
    1635             : 
    1636          26 :   SDValue VecOp = N->getOperand(OpNo);
    1637          26 :   EVT VecVT = VecOp.getValueType();
    1638             :   assert(VecVT.isVector() && "Can only split reduce vector operand");
    1639          13 :   GetSplitVector(VecOp, Lo, Hi);
    1640             :   EVT LoOpVT, HiOpVT;
    1641          39 :   std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
    1642             : 
    1643          26 :   bool NoNaN = N->getFlags().hasNoNaNs();
    1644          13 :   unsigned CombineOpc = 0;
    1645          26 :   switch (N->getOpcode()) {
    1646             :   case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
    1647           0 :   case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
    1648           5 :   case ISD::VECREDUCE_ADD:  CombineOpc = ISD::ADD; break;
    1649           0 :   case ISD::VECREDUCE_MUL:  CombineOpc = ISD::MUL; break;
    1650           0 :   case ISD::VECREDUCE_AND:  CombineOpc = ISD::AND; break;
    1651           0 :   case ISD::VECREDUCE_OR:   CombineOpc = ISD::OR; break;
    1652           0 :   case ISD::VECREDUCE_XOR:  CombineOpc = ISD::XOR; break;
    1653           2 :   case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
    1654           2 :   case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
    1655           2 :   case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
    1656           2 :   case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
    1657           0 :   case ISD::VECREDUCE_FMAX:
    1658           0 :     CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
    1659             :     break;
    1660           0 :   case ISD::VECREDUCE_FMIN:
    1661           0 :     CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
    1662             :     break;
    1663           0 :   default:
    1664           0 :     llvm_unreachable("Unexpected reduce ISD node");
    1665             :   }
    1666             : 
    1667             :   // Use the appropriate scalar instruction on the split subvectors before
    1668             :   // reducing the now partially reduced smaller vector.
    1669          26 :   SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi);
    1670          52 :   return DAG.getNode(N->getOpcode(), dl, ResVT, Partial);
    1671             : }
    1672             : 
    1673        9093 : SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
    1674             :   // The result has a legal vector type, but the input needs splitting.
    1675       18186 :   EVT ResVT = N->getValueType(0);
    1676        9093 :   SDValue Lo, Hi;
    1677       18186 :   SDLoc dl(N);
    1678       18186 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1679       18186 :   EVT InVT = Lo.getValueType();
    1680             : 
    1681        9093 :   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
    1682       18186 :                                InVT.getVectorNumElements());
    1683             : 
    1684       27279 :   Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
    1685       27279 :   Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
    1686             : 
    1687       27279 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
    1688             : }
    1689             : 
    1690        5261 : SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
    1691             :   // For example, i64 = BITCAST v4i16 on alpha.  Typically the vector will
    1692             :   // end up being split all the way down to individual components.  Convert the
    1693             :   // split pieces into integers and reassemble.
    1694        5261 :   SDValue Lo, Hi;
    1695       10522 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1696        5261 :   Lo = BitConvertToInteger(Lo);
    1697        5261 :   Hi = BitConvertToInteger(Hi);
    1698             : 
    1699       10522 :   if (DAG.getDataLayout().isBigEndian())
    1700             :     std::swap(Lo, Hi);
    1701             : 
    1702       15783 :   return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
    1703       21044 :                      JoinIntegers(Lo, Hi));
    1704             : }
    1705             : 
    1706       24547 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
    1707             :   // We know that the extracted result type is legal.
    1708       49094 :   EVT SubVT = N->getValueType(0);
    1709       49094 :   SDValue Idx = N->getOperand(1);
    1710       49094 :   SDLoc dl(N);
    1711       24547 :   SDValue Lo, Hi;
    1712       49094 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1713             : 
    1714       49094 :   uint64_t LoElts = Lo.getValueType().getVectorNumElements();
    1715       49094 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    1716             : 
    1717       24547 :   if (IdxVal < LoElts) {
    1718             :     assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
    1719             :            "Extracted subvector crosses vector split!");
    1720       25364 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
    1721             :   } else {
    1722       11865 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
    1723             :                        DAG.getConstant(IdxVal - LoElts, dl,
    1724       23730 :                                        Idx.getValueType()));
    1725             :   }
    1726             : }
    1727             : 
    1728       13257 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    1729       26514 :   SDValue Vec = N->getOperand(0);
    1730       26514 :   SDValue Idx = N->getOperand(1);
    1731       26514 :   EVT VecVT = Vec.getValueType();
    1732             : 
    1733          96 :   if (isa<ConstantSDNode>(Idx)) {
    1734       26322 :     uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    1735             :     assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
    1736             : 
    1737       13161 :     SDValue Lo, Hi;
    1738       13161 :     GetSplitVector(Vec, Lo, Hi);
    1739             : 
    1740       26322 :     uint64_t LoElts = Lo.getValueType().getVectorNumElements();
    1741             : 
    1742       13161 :     if (IdxVal < LoElts)
    1743        7029 :       return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
    1744       24528 :     return SDValue(DAG.UpdateNodeOperands(N, Hi,
    1745       12264 :                                   DAG.getConstant(IdxVal - LoElts, SDLoc(N),
    1746        6132 :                                                   Idx.getValueType())), 0);
    1747             :   }
    1748             : 
    1749             :   // See if the target wants to custom expand this node.
    1750         192 :   if (CustomLowerNode(N, N->getValueType(0), true))
    1751           0 :     return SDValue();
    1752             : 
    1753             :   // Make the vector elements byte-addressable if they aren't already.
    1754          96 :   SDLoc dl(N);
    1755          96 :   EVT EltVT = VecVT.getVectorElementType();
    1756          96 :   if (EltVT.getSizeInBits() < 8) {
    1757           2 :     SmallVector<SDValue, 4> ElementOps;
    1758           3 :     for (unsigned i = 0; i < VecVT.getVectorNumElements(); ++i) {
    1759           4 :       ElementOps.push_back(DAG.getAnyExtOrTrunc(
    1760           2 :           DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vec,
    1761           4 :                       DAG.getConstant(i, dl, MVT::i8)),
    1762           8 :           dl, MVT::i8));
    1763             :     }
    1764             : 
    1765           1 :     EltVT = MVT::i8;
    1766           1 :     VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
    1767           1 :                              VecVT.getVectorNumElements());
    1768           2 :     Vec = DAG.getBuildVector(VecVT, dl, ElementOps);
    1769             :   }
    1770             : 
    1771             :   // Store the vector to the stack.
    1772          96 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1773             :   SDValue Store =
    1774         384 :       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
    1775             : 
    1776             :   // Load back the required element.
    1777          96 :   StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1778          96 :   return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
    1779         288 :                         MachinePointerInfo(), EltVT);
    1780             : }
    1781             : 
    1782           6 : SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
    1783           6 :   SDValue Lo, Hi;
    1784             : 
    1785             :   // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
    1786             :   // splitting the result has the same effect as splitting the input operand.
    1787           6 :   SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
    1788             : 
    1789          30 :   return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
    1790             : }
    1791             : 
    1792          21 : SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
    1793             :                                              unsigned OpNo) {
    1794          21 :   EVT LoVT, HiVT;
    1795          42 :   SDLoc dl(MGT);
    1796          84 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
    1797             : 
    1798          42 :   SDValue Ch = MGT->getChain();
    1799          42 :   SDValue Ptr = MGT->getBasePtr();
    1800          42 :   SDValue Index = MGT->getIndex();
    1801          42 :   SDValue Mask = MGT->getMask();
    1802          42 :   SDValue Src0 = MGT->getValue();
    1803          42 :   unsigned Alignment = MGT->getOriginalAlignment();
    1804             : 
    1805          21 :   SDValue MaskLo, MaskHi;
    1806          63 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1807             :     // Split Mask operand
    1808           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1809             :   else
    1810          63 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1811             : 
    1812          21 :   EVT MemoryVT = MGT->getMemoryVT();
    1813          21 :   EVT LoMemVT, HiMemVT;
    1814          63 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1815             : 
    1816          21 :   SDValue Src0Lo, Src0Hi;
    1817          63 :   if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
    1818           0 :     GetSplitVector(Src0, Src0Lo, Src0Hi);
    1819             :   else
    1820          63 :     std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
    1821             : 
    1822          21 :   SDValue IndexHi, IndexLo;
    1823          63 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1824          21 :     GetSplitVector(Index, IndexLo, IndexHi);
    1825             :   else
    1826           0 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
    1827             : 
    1828          21 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1829          42 :     getMachineMemOperand(MGT->getPointerInfo(),
    1830          21 :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1831          63 :                          Alignment, MGT->getAAInfo(), MGT->getRanges());
    1832             : 
    1833          21 :   SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
    1834          63 :   SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
    1835          42 :                                    OpsLo, MMO);
    1836             : 
    1837          42 :   MMO = DAG.getMachineFunction().
    1838          42 :     getMachineMemOperand(MGT->getPointerInfo(),
    1839          21 :                          MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),
    1840          63 :                          Alignment, MGT->getAAInfo(),
    1841             :                          MGT->getRanges());
    1842             : 
    1843          21 :   SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
    1844          63 :   SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
    1845          42 :                                    OpsHi, MMO);
    1846             : 
    1847             :   // Build a factor node to remember that this load is independent of the
    1848             :   // other one.
    1849          42 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1850         105 :                    Hi.getValue(1));
    1851             : 
    1852             :   // Legalize the chain result - switch anything that used the old chain to
    1853             :   // use the new one.
    1854          21 :   ReplaceValueWith(SDValue(MGT, 1), Ch);
    1855             : 
    1856          21 :   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
    1857          42 :                             Hi);
    1858          21 :   ReplaceValueWith(SDValue(MGT, 0), Res);
    1859          42 :   return SDValue();
    1860             : }
    1861             : 
    1862           6 : SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
    1863             :                                             unsigned OpNo) {
    1864          12 :   SDValue Ch  = N->getChain();
    1865          12 :   SDValue Ptr = N->getBasePtr();
    1866          12 :   SDValue Mask = N->getMask();
    1867           6 :   SDValue Data = N->getValue();
    1868           6 :   EVT MemoryVT = N->getMemoryVT();
    1869          12 :   unsigned Alignment = N->getOriginalAlignment();
    1870          12 :   SDLoc DL(N);
    1871             : 
    1872           6 :   EVT LoMemVT, HiMemVT;
    1873          18 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1874             : 
    1875           6 :   SDValue DataLo, DataHi;
    1876          18 :   if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
    1877             :     // Split Data operand
    1878           6 :     GetSplitVector(Data, DataLo, DataHi);
    1879             :   else
    1880           0 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    1881             : 
    1882           6 :   SDValue MaskLo, MaskHi;
    1883          18 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1884             :     // Split Mask operand
    1885           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1886             :   else
    1887          18 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
    1888             : 
    1889          12 :   MaskLo = PromoteTargetBoolean(MaskLo, DataLo.getValueType());
    1890          12 :   MaskHi = PromoteTargetBoolean(MaskHi, DataHi.getValueType());
    1891             : 
    1892             :   // if Alignment is equal to the vector size,
    1893             :   // take the half of it for the second part
    1894             :   unsigned SecondHalfAlignment =
    1895          18 :     (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
    1896           6 :        Alignment/2 : Alignment;
    1897             : 
    1898             :   SDValue Lo, Hi;
    1899           6 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1900          12 :     getMachineMemOperand(N->getPointerInfo(),
    1901           6 :                          MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
    1902          18 :                          Alignment, N->getAAInfo(), N->getRanges());
    1903             : 
    1904          12 :   Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
    1905           6 :                           N->isTruncatingStore(),
    1906          18 :                           N->isCompressingStore());
    1907             : 
    1908          12 :   Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    1909          12 :                                    N->isCompressingStore());
    1910          12 :   MMO = DAG.getMachineFunction().
    1911          12 :     getMachineMemOperand(N->getPointerInfo(),
    1912           6 :                          MachineMemOperand::MOStore,  HiMemVT.getStoreSize(),
    1913          18 :                          SecondHalfAlignment, N->getAAInfo(), N->getRanges());
    1914             : 
    1915          12 :   Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
    1916          18 :                           N->isTruncatingStore(), N->isCompressingStore());
    1917             : 
    1918             :   // Build a factor node to remember that this store is independent of the
    1919             :   // other one.
    1920          24 :   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    1921             : }
    1922             : 
    1923          17 : SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
    1924             :                                               unsigned OpNo) {
    1925          34 :   SDValue Ch  = N->getChain();
    1926          34 :   SDValue Ptr = N->getBasePtr();
    1927          34 :   SDValue Mask = N->getMask();
    1928          34 :   SDValue Index = N->getIndex();
    1929          34 :   SDValue Data = N->getValue();
    1930          17 :   EVT MemoryVT = N->getMemoryVT();
    1931          34 :   unsigned Alignment = N->getOriginalAlignment();
    1932          34 :   SDLoc DL(N);
    1933             : 
    1934             :   // Split all operands
    1935          17 :   EVT LoMemVT, HiMemVT;
    1936          51 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1937             : 
    1938          17 :   SDValue DataLo, DataHi;
    1939          51 :   if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
    1940             :     // Split Data operand
    1941          10 :     GetSplitVector(Data, DataLo, DataHi);
    1942             :   else
    1943          21 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    1944             : 
    1945          17 :   SDValue MaskLo, MaskHi;
    1946          51 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1947             :     // Split Mask operand
    1948           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1949             :   else
    1950          51 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
    1951             : 
    1952          17 :   SDValue IndexHi, IndexLo;
    1953          51 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1954          13 :     GetSplitVector(Index, IndexLo, IndexHi);
    1955             :   else
    1956          12 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
    1957             : 
    1958          17 :   SDValue Lo, Hi;
    1959          17 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1960          34 :     getMachineMemOperand(N->getPointerInfo(),
    1961          17 :                          MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
    1962          51 :                          Alignment, N->getAAInfo(), N->getRanges());
    1963             : 
    1964          17 :   SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo};
    1965          68 :   Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
    1966          51 :                             DL, OpsLo, MMO);
    1967             : 
    1968          34 :   MMO = DAG.getMachineFunction().
    1969          34 :     getMachineMemOperand(N->getPointerInfo(),
    1970          17 :                          MachineMemOperand::MOStore,  HiMemVT.getStoreSize(),
    1971          51 :                          Alignment, N->getAAInfo(), N->getRanges());
    1972             : 
    1973             :   // The order of the Scatter operation after split is well defined. The "Hi"
    1974             :   // part comes after the "Lo". So these two operations should be chained one
    1975             :   // after another.
    1976          17 :   SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi};
    1977          51 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
    1978          51 :                               DL, OpsHi, MMO);
    1979             : }
    1980             : 
    1981        6847 : SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
    1982             :   assert(N->isUnindexed() && "Indexed store of vector?");
    1983             :   assert(OpNo == 1 && "Can only split the stored value");
    1984       13694 :   SDLoc DL(N);
    1985             : 
    1986       13694 :   bool isTruncating = N->isTruncatingStore();
    1987       13694 :   SDValue Ch  = N->getChain();
    1988        6847 :   SDValue Ptr = N->getBasePtr();
    1989        6847 :   EVT MemoryVT = N->getMemoryVT();
    1990       13694 :   unsigned Alignment = N->getOriginalAlignment();
    1991        6847 :   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
    1992        6847 :   AAMDNodes AAInfo = N->getAAInfo();
    1993        6847 :   SDValue Lo, Hi;
    1994       13694 :   GetSplitVector(N->getOperand(1), Lo, Hi);
    1995             : 
    1996        6847 :   EVT LoMemVT, HiMemVT;
    1997       20541 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1998             : 
    1999        6847 :   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
    2000             : 
    2001        6847 :   if (isTruncating)
    2002           0 :     Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
    2003           0 :                            Alignment, MMOFlags, AAInfo);
    2004             :   else
    2005       20541 :     Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
    2006        6847 :                       AAInfo);
    2007             : 
    2008             :   // Increment the pointer to the other half.
    2009       13694 :   Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
    2010       20541 :                     DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
    2011             : 
    2012        6847 :   if (isTruncating)
    2013           0 :     Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
    2014           0 :                            N->getPointerInfo().getWithOffset(IncrementSize),
    2015           0 :                            HiMemVT, Alignment, MMOFlags, AAInfo);
    2016             :   else
    2017       13694 :     Hi = DAG.getStore(Ch, DL, Hi, Ptr,
    2018        6847 :                       N->getPointerInfo().getWithOffset(IncrementSize),
    2019       20541 :                       Alignment, MMOFlags, AAInfo);
    2020             : 
    2021       27388 :   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    2022             : }
    2023             : 
    2024           0 : SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
    2025           0 :   SDLoc DL(N);
    2026             : 
    2027             :   // The input operands all must have the same type, and we know the result
    2028             :   // type is valid.  Convert this to a buildvector which extracts all the
    2029             :   // input elements.
    2030             :   // TODO: If the input elements are power-two vectors, we could convert this to
    2031             :   // a new CONCAT_VECTORS node with elements that are half-wide.
    2032           0 :   SmallVector<SDValue, 32> Elts;
    2033           0 :   EVT EltVT = N->getValueType(0).getVectorElementType();
    2034           0 :   for (const SDValue &Op : N->op_values()) {
    2035           0 :     for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
    2036           0 :          i != e; ++i) {
    2037           0 :       Elts.push_back(DAG.getNode(
    2038             :           ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
    2039           0 :           DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    2040             :     }
    2041             :   }
    2042             : 
    2043           0 :   return DAG.getBuildVector(N->getValueType(0), DL, Elts);
    2044             : }
    2045             : 
    2046         804 : SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
    2047             :   // The result type is legal, but the input type is illegal.  If splitting
    2048             :   // ends up with the result type of each half still being legal, just
    2049             :   // do that.  If, however, that would result in an illegal result type,
    2050             :   // we can try to get more clever with power-two vectors. Specifically,
    2051             :   // split the input type, but also widen the result element size, then
    2052             :   // concatenate the halves and truncate again.  For example, consider a target
    2053             :   // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
    2054             :   // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
    2055             :   //   %inlo = v4i32 extract_subvector %in, 0
    2056             :   //   %inhi = v4i32 extract_subvector %in, 4
    2057             :   //   %lo16 = v4i16 trunc v4i32 %inlo
    2058             :   //   %hi16 = v4i16 trunc v4i32 %inhi
    2059             :   //   %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
    2060             :   //   %res = v8i8 trunc v8i16 %in16
    2061             :   //
    2062             :   // Without this transform, the original truncate would end up being
    2063             :   // scalarized, which is pretty much always a last resort.
    2064        1608 :   SDValue InVec = N->getOperand(0);
    2065        1608 :   EVT InVT = InVec->getValueType(0);
    2066        1608 :   EVT OutVT = N->getValueType(0);
    2067         804 :   unsigned NumElements = OutVT.getVectorNumElements();
    2068         804 :   bool IsFloat = OutVT.isFloatingPoint();
    2069             : 
    2070             :   // Widening should have already made sure this is a power-two vector
    2071             :   // if we're trying to split it at all. assert() that's true, just in case.
    2072             :   assert(!(NumElements & 1) && "Splitting vector, but not in half!");
    2073             : 
    2074         804 :   unsigned InElementSize = InVT.getScalarSizeInBits();
    2075         804 :   unsigned OutElementSize = OutVT.getScalarSizeInBits();
    2076             : 
    2077             :   // If the input elements are only 1/2 the width of the result elements,
    2078             :   // just use the normal splitting. Our trick only work if there's room
    2079             :   // to split more than once.
    2080         804 :   if (InElementSize <= OutElementSize * 2)
    2081         590 :     return SplitVecOp_UnaryOp(N);
    2082         214 :   SDLoc DL(N);
    2083             : 
    2084             :   // Extract the halves of the input via extract_subvector.
    2085             :   SDValue InLoVec, InHiVec;
    2086         642 :   std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL);
    2087             :   // Truncate them to 1/2 the element size.
    2088             :   EVT HalfElementVT = IsFloat ?
    2089          17 :     EVT::getFloatingPointVT(InElementSize/2) :
    2090         231 :     EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
    2091         214 :   EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
    2092         428 :                                 NumElements/2);
    2093         642 :   SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
    2094         642 :   SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
    2095             :   // Concatenate them to get the full intermediate truncation result.
    2096         214 :   EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
    2097         214 :   SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
    2098         214 :                                  HalfHi);
    2099             :   // Now finish up by truncating all the way down to the original result
    2100             :   // type. This should normally be something that ends up being legal directly,
    2101             :   // but in theory if a target has very wide vectors and an annoyingly
    2102             :   // restricted set of legal types, this split can chain to build things up.
    2103             :   return IsFloat
    2104          17 :              ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
    2105             :                            DAG.getTargetConstant(
    2106         282 :                                0, DL, TLI.getPointerTy(DAG.getDataLayout())))
    2107         428 :              : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
    2108             : }
    2109             : 
    2110        1048 : SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
    2111             :   assert(N->getValueType(0).isVector() &&
    2112             :          N->getOperand(0).getValueType().isVector() &&
    2113             :          "Operand types must be vectors");
    2114             :   // The result has a legal vector type, but the input needs splitting.
    2115        1048 :   SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
    2116        2096 :   SDLoc DL(N);
    2117        2096 :   GetSplitVector(N->getOperand(0), Lo0, Hi0);
    2118        2096 :   GetSplitVector(N->getOperand(1), Lo1, Hi1);
    2119        2096 :   unsigned PartElements = Lo0.getValueType().getVectorNumElements();
    2120        1048 :   EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
    2121        2096 :   EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
    2122             : 
    2123        2096 :   LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
    2124        2096 :   HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
    2125        2096 :   SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
    2126        3144 :   return PromoteTargetBoolean(Con, N->getValueType(0));
    2127             : }
    2128             : 
    2129             : 
    2130          73 : SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
    2131             :   // The result has a legal vector type, but the input needs splitting.
    2132         146 :   EVT ResVT = N->getValueType(0);
    2133          73 :   SDValue Lo, Hi;
    2134         146 :   SDLoc DL(N);
    2135         146 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    2136         146 :   EVT InVT = Lo.getValueType();
    2137             : 
    2138          73 :   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
    2139         146 :                                InVT.getVectorNumElements());
    2140             : 
    2141         219 :   Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
    2142         219 :   Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
    2143             : 
    2144         219 :   return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
    2145             : }
    2146             : 
    2147           8 : SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
    2148             :   // The result (and the first input) has a legal vector type, but the second
    2149             :   // input needs splitting.
    2150          16 :   return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
    2151             : }
    2152             : 
    2153             : 
    2154             : //===----------------------------------------------------------------------===//
    2155             : //  Result Vector Widening
    2156             : //===----------------------------------------------------------------------===//
    2157             : 
    2158        5914 : void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
    2159             :   DEBUG(dbgs() << "Widen node result " << ResNo << ": ";
    2160             :         N->dump(&DAG);
    2161             :         dbgs() << "\n");
    2162             : 
    2163             :   // See if the target wants to custom widen this node.
    2164       11828 :   if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
    2165          79 :     return;
    2166             : 
    2167        5835 :   SDValue Res = SDValue();
    2168       11670 :   switch (N->getOpcode()) {
    2169           0 :   default:
    2170             : #ifndef NDEBUG
    2171             :     dbgs() << "WidenVectorResult #" << ResNo << ": ";
    2172             :     N->dump(&DAG);
    2173             :     dbgs() << "\n";
    2174             : #endif
    2175           0 :     llvm_unreachable("Do not know how to widen the result of this operator!");
    2176             : 
    2177           0 :   case ISD::MERGE_VALUES:      Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
    2178         298 :   case ISD::BITCAST:           Res = WidenVecRes_BITCAST(N); break;
    2179         439 :   case ISD::BUILD_VECTOR:      Res = WidenVecRes_BUILD_VECTOR(N); break;
    2180          35 :   case ISD::CONCAT_VECTORS:    Res = WidenVecRes_CONCAT_VECTORS(N); break;
    2181        2359 :   case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
    2182           0 :   case ISD::FP_ROUND_INREG:    Res = WidenVecRes_InregOp(N); break;
    2183          25 :   case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
    2184         563 :   case ISD::LOAD:              Res = WidenVecRes_LOAD(N); break;
    2185           4 :   case ISD::SCALAR_TO_VECTOR:  Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
    2186          11 :   case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
    2187         174 :   case ISD::VSELECT:
    2188         174 :   case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
    2189           0 :   case ISD::SELECT_CC:         Res = WidenVecRes_SELECT_CC(N); break;
    2190          34 :   case ISD::SETCC:             Res = WidenVecRes_SETCC(N); break;
    2191         493 :   case ISD::UNDEF:             Res = WidenVecRes_UNDEF(N); break;
    2192          50 :   case ISD::VECTOR_SHUFFLE:
    2193          50 :     Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
    2194          50 :     break;
    2195          10 :   case ISD::MLOAD:
    2196          10 :     Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
    2197          10 :     break;
    2198          15 :   case ISD::MGATHER:
    2199          15 :     Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
    2200          15 :     break;
    2201             : 
    2202         141 :   case ISD::ADD:
    2203             :   case ISD::AND:
    2204             :   case ISD::MUL:
    2205             :   case ISD::MULHS:
    2206             :   case ISD::MULHU:
    2207             :   case ISD::OR:
    2208             :   case ISD::SUB:
    2209             :   case ISD::XOR:
    2210             :   case ISD::FMINNUM:
    2211             :   case ISD::FMAXNUM:
    2212             :   case ISD::FMINNAN:
    2213             :   case ISD::FMAXNAN:
    2214             :   case ISD::SMIN:
    2215             :   case ISD::SMAX:
    2216             :   case ISD::UMIN:
    2217             :   case ISD::UMAX:
    2218         141 :     Res = WidenVecRes_Binary(N);
    2219         141 :     break;
    2220             : 
    2221          93 :   case ISD::FADD:
    2222             :   case ISD::FMUL:
    2223             :   case ISD::FPOW:
    2224             :   case ISD::FSUB:
    2225             :   case ISD::FDIV:
    2226             :   case ISD::FREM:
    2227             :   case ISD::SDIV:
    2228             :   case ISD::UDIV:
    2229             :   case ISD::SREM:
    2230             :   case ISD::UREM:
    2231          93 :     Res = WidenVecRes_BinaryCanTrap(N);
    2232          93 :     break;
    2233             : 
    2234           5 :   case ISD::FCOPYSIGN:
    2235           5 :     Res = WidenVecRes_FCOPYSIGN(N);
    2236           5 :     break;
    2237             : 
    2238           1 :   case ISD::FPOWI:
    2239           1 :     Res = WidenVecRes_POWI(N);
    2240           1 :     break;
    2241             : 
    2242          16 :   case ISD::SHL:
    2243             :   case ISD::SRA:
    2244             :   case ISD::SRL:
    2245          16 :     Res = WidenVecRes_Shift(N);
    2246          16 :     break;
    2247             : 
    2248           8 :   case ISD::ANY_EXTEND_VECTOR_INREG:
    2249             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
    2250             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
    2251           8 :     Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
    2252           8 :     break;
    2253             : 
    2254        1048 :   case ISD::ANY_EXTEND:
    2255             :   case ISD::FP_EXTEND:
    2256             :   case ISD::FP_ROUND:
    2257             :   case ISD::FP_TO_SINT:
    2258             :   case ISD::FP_TO_UINT:
    2259             :   case ISD::SIGN_EXTEND:
    2260             :   case ISD::SINT_TO_FP:
    2261             :   case ISD::TRUNCATE:
    2262             :   case ISD::UINT_TO_FP:
    2263             :   case ISD::ZERO_EXTEND:
    2264        1048 :     Res = WidenVecRes_Convert(N);
    2265        1048 :     break;
    2266             : 
    2267          11 :   case ISD::BITREVERSE:
    2268             :   case ISD::BSWAP:
    2269             :   case ISD::CTLZ:
    2270             :   case ISD::CTPOP:
    2271             :   case ISD::CTTZ:
    2272             :   case ISD::FABS:
    2273             :   case ISD::FCEIL:
    2274             :   case ISD::FCOS:
    2275             :   case ISD::FEXP:
    2276             :   case ISD::FEXP2:
    2277             :   case ISD::FFLOOR:
    2278             :   case ISD::FLOG:
    2279             :   case ISD::FLOG10:
    2280             :   case ISD::FLOG2:
    2281             :   case ISD::FNEARBYINT:
    2282             :   case ISD::FNEG:
    2283             :   case ISD::FRINT:
    2284             :   case ISD::FROUND:
    2285             :   case ISD::FSIN:
    2286             :   case ISD::FSQRT:
    2287             :   case ISD::FTRUNC:
    2288          11 :     Res = WidenVecRes_Unary(N);
    2289          11 :     break;
    2290           2 :   case ISD::FMA:
    2291           2 :     Res = WidenVecRes_Ternary(N);
    2292           2 :     break;
    2293             :   }
    2294             : 
    2295             :   // If Res is null, the sub-method took care of registering the result.
    2296        5835 :   if (Res.getNode())
    2297        5835 :     SetWidenedVector(SDValue(N, ResNo), Res);
    2298             : }
    2299             : 
    2300           2 : SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
    2301             :   // Ternary op widening.
    2302           4 :   SDLoc dl(N);
    2303           6 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2304           4 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2305           4 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2306           4 :   SDValue InOp3 = GetWidenedVector(N->getOperand(2));
    2307           6 :   return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
    2308             : }
    2309             : 
    2310         141 : SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
    2311             :   // Binary op widening.
    2312         282 :   SDLoc dl(N);
    2313         423 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2314         282 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2315         282 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2316         564 :   return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
    2317             : }
    2318             : 
    2319          97 : SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
    2320             :   // Binary op widening for operations that can trap.
    2321         194 :   unsigned Opcode = N->getOpcode();
    2322         194 :   SDLoc dl(N);
    2323         291 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2324          97 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    2325          97 :   EVT VT = WidenVT;
    2326          97 :   unsigned NumElts =  VT.getVectorNumElements();
    2327         194 :   const SDNodeFlags Flags = N->getFlags();
    2328         175 :   while (!TLI.isTypeLegal(VT) && NumElts != 1) {
    2329          24 :     NumElts = NumElts / 2;
    2330          24 :     VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2331             :   }
    2332             : 
    2333         279 :   if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
    2334             :     // Operation doesn't trap so just widen as normal.
    2335         142 :     SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2336         142 :     SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2337         142 :     return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
    2338             :   }
    2339             : 
    2340             :   // No legal vector version so unroll the vector operation and then widen.
    2341          26 :   if (NumElts == 1)
    2342           6 :     return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
    2343             : 
    2344             :   // Since the operation can trap, apply operation on the original vector.
    2345          20 :   EVT MaxVT = VT;
    2346          40 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2347          40 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2348          40 :   unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
    2349             : 
    2350          40 :   SmallVector<SDValue, 16> ConcatOps(CurNumElts);
    2351          20 :   unsigned ConcatEnd = 0;  // Current ConcatOps index.
    2352          20 :   int Idx = 0;        // Current Idx into input vectors.
    2353             : 
    2354             :   // NumElts := greatest legal vector size (at most WidenVT)
    2355             :   // while (orig. vector has unhandled elements) {
    2356             :   //   take munches of size NumElts from the beginning and add to ConcatOps
    2357             :   //   NumElts := next smaller supported vector size or 1
    2358             :   // }
    2359          40 :   while (CurNumElts != 0) {
    2360          28 :     while (CurNumElts >= NumElts) {
    2361           4 :       SDValue EOp1 = DAG.getNode(
    2362             :           ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
    2363          12 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2364           4 :       SDValue EOp2 = DAG.getNode(
    2365             :           ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
    2366          12 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2367           8 :       ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
    2368           4 :       Idx += NumElts;
    2369           4 :       CurNumElts -= NumElts;
    2370             :     }
    2371             :     do {
    2372          41 :       NumElts = NumElts / 2;
    2373          41 :       VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2374          82 :     } while (!TLI.isTypeLegal(VT) && NumElts != 1);
    2375             : 
    2376          20 :     if (NumElts == 1) {
    2377          90 :       for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
    2378          35 :         SDValue EOp1 = DAG.getNode(
    2379             :             ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
    2380         105 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2381          35 :         SDValue EOp2 = DAG.getNode(
    2382             :             ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
    2383         105 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2384          70 :         ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
    2385             :                                              EOp1, EOp2, Flags);
    2386             :       }
    2387             :       CurNumElts = 0;
    2388             :     }
    2389             :   }
    2390             : 
    2391             :   // Check to see if we have a single operation with the widen type.
    2392          20 :   if (ConcatEnd == 1) {
    2393          24 :     VT = ConcatOps[0].getValueType();
    2394          12 :     if (VT == WidenVT)
    2395           0 :       return ConcatOps[0];
    2396             :   }
    2397             : 
    2398             :   // while (Some element of ConcatOps is not of type MaxVT) {
    2399             :   //   From the end of ConcatOps, collect elements of the same type and put
    2400             :   //   them into an op of the next larger supported type
    2401             :   // }
    2402         140 :   while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
    2403          20 :     Idx = ConcatEnd - 1;
    2404          60 :     VT = ConcatOps[Idx--].getValueType();
    2405          86 :     while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
    2406          15 :       Idx--;
    2407             : 
    2408          20 :     int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
    2409          20 :     EVT NextVT;
    2410             :     do {
    2411          41 :       NextSize *= 2;
    2412          41 :       NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
    2413          61 :     } while (!TLI.isTypeLegal(NextVT));
    2414             : 
    2415          20 :     if (!VT.isVector()) {
    2416             :       // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
    2417          20 :       SDValue VecOp = DAG.getUNDEF(NextVT);
    2418          20 :       unsigned NumToInsert = ConcatEnd - Idx - 1;
    2419          55 :       for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
    2420          70 :         VecOp = DAG.getNode(
    2421          70 :             ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
    2422         140 :             DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2423             :       }
    2424          40 :       ConcatOps[Idx+1] = VecOp;
    2425          20 :       ConcatEnd = Idx + 2;
    2426             :     } else {
    2427             :       // Vector type, create a CONCAT_VECTORS of type NextVT
    2428           0 :       SDValue undefVec = DAG.getUNDEF(VT);
    2429           0 :       unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
    2430           0 :       SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
    2431           0 :       unsigned RealVals = ConcatEnd - Idx - 1;
    2432           0 :       unsigned SubConcatEnd = 0;
    2433           0 :       unsigned SubConcatIdx = Idx + 1;
    2434           0 :       while (SubConcatEnd < RealVals)
    2435           0 :         SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
    2436           0 :       while (SubConcatEnd < OpsToConcat)
    2437           0 :         SubConcatOps[SubConcatEnd++] = undefVec;
    2438           0 :       ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
    2439           0 :                                             NextVT, SubConcatOps);
    2440           0 :       ConcatEnd = SubConcatIdx + 1;
    2441             :     }
    2442             :   }
    2443             : 
    2444             :   // Check to see if we have a single operation with the widen type.
    2445          20 :   if (ConcatEnd == 1) {
    2446          34 :     VT = ConcatOps[0].getValueType();
    2447          17 :     if (VT == WidenVT)
    2448          17 :       return ConcatOps[0];
    2449             :   }
    2450             : 
    2451             :   // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
    2452           3 :   unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
    2453           3 :   if (NumOps != ConcatEnd ) {
    2454           1 :     SDValue UndefVal = DAG.getUNDEF(MaxVT);
    2455           2 :     for (unsigned j = ConcatEnd; j < NumOps; ++j)
    2456           2 :       ConcatOps[j] = UndefVal;
    2457             :   }
    2458           3 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
    2459           9 :                      makeArrayRef(ConcatOps.data(), NumOps));
    2460             : }
    2461             : 
    2462        1048 : SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
    2463        2096 :   SDValue InOp = N->getOperand(0);
    2464        2096 :   SDLoc DL(N);
    2465             : 
    2466        3144 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2467        1048 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2468             : 
    2469        2096 :   EVT InVT = InOp.getValueType();
    2470        1048 :   EVT InEltVT = InVT.getVectorElementType();
    2471        1048 :   EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
    2472             : 
    2473        2096 :   unsigned Opcode = N->getOpcode();
    2474        1048 :   unsigned InVTNumElts = InVT.getVectorNumElements();
    2475        2096 :   const SDNodeFlags Flags = N->getFlags();
    2476        2096 :   if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
    2477         360 :     InOp = GetWidenedVector(N->getOperand(0));
    2478         360 :     InVT = InOp.getValueType();
    2479         180 :     InVTNumElts = InVT.getVectorNumElements();
    2480         180 :     if (InVTNumElts == WidenNumElts) {
    2481         171 :       if (N->getNumOperands() == 1)
    2482         336 :         return DAG.getNode(Opcode, DL, WidenVT, InOp);
    2483           6 :       return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
    2484             :     }
    2485           9 :     if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
    2486             :       // If both input and result vector types are of same width, extend
    2487             :       // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
    2488             :       // accepts fewer elements in the result than in the input.
    2489           9 :       if (Opcode == ISD::SIGN_EXTEND)
    2490           4 :         return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
    2491           5 :       if (Opcode == ISD::ZERO_EXTEND)
    2492           5 :         return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
    2493             :     }
    2494             :   }
    2495             : 
    2496         913 :   if (TLI.isTypeLegal(InWidenVT)) {
    2497             :     // Because the result and the input are different vector types, widening
    2498             :     // the result could create a legal type but widening the input might make
    2499             :     // it an illegal type that might lead to repeatedly splitting the input
    2500             :     // and then widening it. To avoid this, we widen the input only if
    2501             :     // it results in a legal type.
    2502          45 :     if (WidenNumElts % InVTNumElts == 0) {
    2503             :       // Widen the input and call convert on the widened input vector.
    2504          45 :       unsigned NumConcat = WidenNumElts/InVTNumElts;
    2505         135 :       SmallVector<SDValue, 16> Ops(NumConcat);
    2506          45 :       Ops[0] = InOp;
    2507          45 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    2508          90 :       for (unsigned i = 1; i != NumConcat; ++i)
    2509          90 :         Ops[i] = UndefVal;
    2510         135 :       SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
    2511          45 :       if (N->getNumOperands() == 1)
    2512          90 :         return DAG.getNode(Opcode, DL, WidenVT, InVec);
    2513           0 :       return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
    2514             :     }
    2515             : 
    2516           0 :     if (InVTNumElts % WidenNumElts == 0) {
    2517           0 :       SDValue InVal = DAG.getNode(
    2518             :           ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
    2519           0 :           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2520             :       // Extract the input and convert the shorten input vector.
    2521           0 :       if (N->getNumOperands() == 1)
    2522           0 :         return DAG.getNode(Opcode, DL, WidenVT, InVal);
    2523           0 :       return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
    2524             :     }
    2525             :   }
    2526             : 
    2527             :   // Otherwise unroll into some nasty scalar code and rebuild the vector.
    2528        1646 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    2529         823 :   EVT EltVT = WidenVT.getVectorElementType();
    2530         823 :   unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
    2531             :   unsigned i;
    2532        3075 :   for (i=0; i < MinElts; ++i) {
    2533        2252 :     SDValue Val = DAG.getNode(
    2534             :         ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
    2535        6756 :         DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2536        2252 :     if (N->getNumOperands() == 1)
    2537        6684 :       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
    2538             :     else
    2539          72 :       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
    2540             :   }
    2541             : 
    2542         823 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    2543        5807 :   for (; i < WidenNumElts; ++i)
    2544        4984 :     Ops[i] = UndefVal;
    2545             : 
    2546        1646 :   return DAG.getBuildVector(WidenVT, DL, Ops);
    2547             : }
    2548             : 
    2549           8 : SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
    2550          16 :   unsigned Opcode = N->getOpcode();
    2551          16 :   SDValue InOp = N->getOperand(0);
    2552          16 :   SDLoc DL(N);
    2553             : 
    2554          24 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2555           8 :   EVT WidenSVT = WidenVT.getVectorElementType();
    2556           8 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2557             : 
    2558          16 :   EVT InVT = InOp.getValueType();
    2559           8 :   EVT InSVT = InVT.getVectorElementType();
    2560           8 :   unsigned InVTNumElts = InVT.getVectorNumElements();
    2561             : 
    2562          16 :   if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
    2563           8 :     InOp = GetWidenedVector(InOp);
    2564          16 :     InVT = InOp.getValueType();
    2565           8 :     if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
    2566           8 :       switch (Opcode) {
    2567           0 :       case ISD::ANY_EXTEND_VECTOR_INREG:
    2568           0 :         return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
    2569           3 :       case ISD::SIGN_EXTEND_VECTOR_INREG:
    2570           3 :         return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
    2571           5 :       case ISD::ZERO_EXTEND_VECTOR_INREG:
    2572           5 :         return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
    2573             :       }
    2574             :     }
    2575             :   }
    2576             : 
    2577             :   // Unroll, extend the scalars and rebuild the vector.
    2578           0 :   SmallVector<SDValue, 16> Ops;
    2579           0 :   for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
    2580           0 :     SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
    2581           0 :       DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2582           0 :     switch (Opcode) {
    2583           0 :     case ISD::ANY_EXTEND_VECTOR_INREG:
    2584           0 :       Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
    2585           0 :       break;
    2586           0 :     case ISD::SIGN_EXTEND_VECTOR_INREG:
    2587           0 :       Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
    2588           0 :       break;
    2589           0 :     case ISD::ZERO_EXTEND_VECTOR_INREG:
    2590           0 :       Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
    2591           0 :       break;
    2592           0 :     default:
    2593           0 :       llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
    2594             :     }
    2595           0 :     Ops.push_back(Val);
    2596             :   }
    2597             : 
    2598           0 :   while (Ops.size() != WidenNumElts)
    2599           0 :     Ops.push_back(DAG.getUNDEF(WidenSVT));
    2600             : 
    2601           0 :   return DAG.getBuildVector(WidenVT, DL, Ops);
    2602             : }
    2603             : 
    2604           5 : SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
    2605             :   // If this is an FCOPYSIGN with same input types, we can treat it as a
    2606             :   // normal (can trap) binary op.
    2607          28 :   if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
    2608           4 :     return WidenVecRes_BinaryCanTrap(N);
    2609             : 
    2610             :   // If the types are different, fall back to unrolling.
    2611           3 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2612           1 :   return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
    2613             : }
    2614             : 
    2615           1 : SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
    2616           3 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2617           2 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2618           2 :   SDValue ShOp = N->getOperand(1);
    2619           5 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
    2620             : }
    2621             : 
    2622          16 : SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
    2623          48 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2624          32 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2625          32 :   SDValue ShOp = N->getOperand(1);
    2626             : 
    2627          32 :   EVT ShVT = ShOp.getValueType();
    2628          32 :   if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
    2629          16 :     ShOp = GetWidenedVector(ShOp);
    2630          32 :     ShVT = ShOp.getValueType();
    2631             :   }
    2632          16 :   EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
    2633             :                                    ShVT.getVectorElementType(),
    2634          32 :                                    WidenVT.getVectorNumElements());
    2635          16 :   if (ShVT != ShWidenVT)
    2636           0 :     ShOp = ModifyToType(ShOp, ShWidenVT);
    2637             : 
    2638          80 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
    2639             : }
    2640             : 
    2641          11 : SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
    2642             :   // Unary op widening.
    2643          33 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2644          22 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2645          55 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
    2646             : }
    2647             : 
    2648          11 : SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
    2649          33 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2650          11 :   EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
    2651          44 :                                cast<VTSDNode>(N->getOperand(1))->getVT()
    2652             :                                  .getVectorElementType(),
    2653          33 :                                WidenVT.getVectorNumElements());
    2654          22 :   SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
    2655          33 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
    2656          44 :                      WidenVT, WidenLHS, DAG.getValueType(ExtVT));
    2657             : }
    2658             : 
    2659           0 : SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
    2660           0 :   SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
    2661           0 :   return GetWidenedVector(WidenVec);
    2662             : }
    2663             : 
    2664         298 : SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
    2665         596 :   SDValue InOp = N->getOperand(0);
    2666         596 :   EVT InVT = InOp.getValueType();
    2667         596 :   EVT VT = N->getValueType(0);
    2668         596 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2669         596 :   SDLoc dl(N);
    2670             : 
    2671         596 :   switch (getTypeAction(InVT)) {
    2672             :   case TargetLowering::TypeLegal:
    2673             :     break;
    2674          26 :   case TargetLowering::TypePromoteInteger:
    2675             :     // If the incoming type is a vector that is being promoted, then
    2676             :     // we know that the elements are arranged differently and that we
    2677             :     // must perform the conversion using a stack slot.
    2678          26 :     if (InVT.isVector())
    2679             :       break;
    2680             : 
    2681             :     // If the InOp is promoted to the same size, convert it.  Otherwise,
    2682             :     // fall out of the switch and widen the promoted input.
    2683          14 :     InOp = GetPromotedInteger(InOp);
    2684          28 :     InVT = InOp.getValueType();
    2685          14 :     if (WidenVT.bitsEq(InVT))
    2686          12 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
    2687             :     break;
    2688             :   case TargetLowering::TypeSoftenFloat:
    2689             :   case TargetLowering::TypePromoteFloat:
    2690             :   case TargetLowering::TypeExpandInteger:
    2691             :   case TargetLowering::TypeExpandFloat:
    2692             :   case TargetLowering::TypeScalarizeVector:
    2693             :   case TargetLowering::TypeSplitVector:
    2694             :     break;
    2695         210 :   case TargetLowering::TypeWidenVector:
    2696             :     // If the InOp is widened to the same size, convert it.  Otherwise, fall
    2697             :     // out of the switch and widen the widened input.
    2698         210 :     InOp = GetWidenedVector(InOp);
    2699         420 :     InVT = InOp.getValueType();
    2700         210 :     if (WidenVT.bitsEq(InVT))
    2701             :       // The input widens to the same size. Convert to the widen value.
    2702         420 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
    2703             :     break;
    2704             :   }
    2705             : 
    2706          82 :   unsigned WidenSize = WidenVT.getSizeInBits();
    2707          82 :   unsigned InSize = InVT.getSizeInBits();
    2708             :   // x86mmx is not an acceptable vector element type, so don't try.
    2709         156 :   if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
    2710             :     // Determine new input vector type.  The new input vector type will use
    2711             :     // the same element type (if its a vector) or use the input type as a
    2712             :     // vector.  It is the same size as the type to widen to.
    2713          74 :     EVT NewInVT;
    2714          74 :     unsigned NewNumElts = WidenSize / InSize;
    2715          74 :     if (InVT.isVector()) {
    2716          12 :       EVT InEltVT = InVT.getVectorElementType();
    2717          12 :       NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
    2718          12 :                                  WidenSize / InEltVT.getSizeInBits());
    2719             :     } else {
    2720          62 :       NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
    2721             :     }
    2722             : 
    2723         146 :     if (TLI.isTypeLegal(NewInVT)) {
    2724             :       // Because the result and the input are different vector types, widening
    2725             :       // the result could create a legal type but widening the input might make
    2726             :       // it an illegal type that might lead to repeatedly splitting the input
    2727             :       // and then widening it. To avoid this, we widen the input only if
    2728             :       // it results in a legal type.
    2729         216 :       SmallVector<SDValue, 16> Ops(NewNumElts);
    2730          72 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    2731          72 :       Ops[0] = InOp;
    2732         192 :       for (unsigned i = 1; i < NewNumElts; ++i)
    2733         240 :         Ops[i] = UndefVal;
    2734             : 
    2735          72 :       SDValue NewVec;
    2736          72 :       if (InVT.isVector())
    2737          36 :         NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
    2738             :       else
    2739         120 :         NewVec = DAG.getBuildVector(NewInVT, dl, Ops);
    2740         144 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
    2741             :     }
    2742             :   }
    2743             : 
    2744          10 :   return CreateStackStoreLoad(InOp, WidenVT);
    2745             : }
    2746             : 
    2747         439 : SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
    2748         878 :   SDLoc dl(N);
    2749             :   // Build a vector with undefined for the new nodes.
    2750         878 :   EVT VT = N->getValueType(0);
    2751             : 
    2752             :   // Integer BUILD_VECTOR operands may be larger than the node's vector element
    2753             :   // type. The UNDEFs need to have the same type as the existing operands.
    2754        1317 :   EVT EltVT = N->getOperand(0).getValueType();
    2755         439 :   unsigned NumElts = VT.getVectorNumElements();
    2756             : 
    2757         878 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2758         439 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2759             : 
    2760        1756 :   SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
    2761             :   assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
    2762         439 :   NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
    2763             : 
    2764        1317 :   return DAG.getBuildVector(WidenVT, dl, NewOps);
    2765             : }
    2766             : 
    2767          35 : SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
    2768         105 :   EVT InVT = N->getOperand(0).getValueType();
    2769         105 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2770          70 :   SDLoc dl(N);
    2771          35 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2772          35 :   unsigned NumInElts = InVT.getVectorNumElements();
    2773          70 :   unsigned NumOperands = N->getNumOperands();
    2774             : 
    2775          35 :   bool InputWidened = false; // Indicates we need to widen the input.
    2776          70 :   if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
    2777           9 :     if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
    2778             :       // Add undef vectors to widen to correct length.
    2779             :       unsigned NumConcat = WidenVT.getVectorNumElements() /
    2780           9 :                            InVT.getVectorNumElements();
    2781           9 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    2782          27 :       SmallVector<SDValue, 16> Ops(NumConcat);
    2783          30 :       for (unsigned i=0; i < NumOperands; ++i)
    2784          63 :         Ops[i] = N->getOperand(i);
    2785          39 :       for (unsigned i = NumOperands; i != NumConcat; ++i)
    2786          30 :         Ops[i] = UndefVal;
    2787          27 :       return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
    2788             :     }
    2789             :   } else {
    2790          26 :     InputWidened = true;
    2791          52 :     if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
    2792             :       // The inputs and the result are widen to the same value.
    2793             :       unsigned i;
    2794           4 :       for (i=1; i < NumOperands; ++i)
    2795          12 :         if (!N->getOperand(i).isUndef())
    2796             :           break;
    2797             : 
    2798           4 :       if (i == NumOperands)
    2799             :         // Everything but the first operand is an UNDEF so just return the
    2800             :         // widened first operand.
    2801           0 :         return GetWidenedVector(N->getOperand(0));
    2802             : 
    2803           4 :       if (NumOperands == 2) {
    2804             :         // Replace concat of two operands with a shuffle.
    2805          12 :         SmallVector<int, 16> MaskOps(WidenNumElts, -1);
    2806          16 :         for (unsigned i = 0; i < NumInElts; ++i) {
    2807          24 :           MaskOps[i] = i;
    2808          24 :           MaskOps[i + NumInElts] = i + WidenNumElts;
    2809             :         }
    2810           4 :         return DAG.getVectorShuffle(WidenVT, dl,
    2811           8 :                                     GetWidenedVector(N->getOperand(0)),
    2812           8 :                                     GetWidenedVector(N->getOperand(1)),
    2813          12 :                                     MaskOps);
    2814             :       }
    2815             :     }
    2816             :   }
    2817             : 
    2818             :   // Fall back to use extracts and build vector.
    2819          22 :   EVT EltVT = WidenVT.getVectorElementType();
    2820          44 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    2821          22 :   unsigned Idx = 0;
    2822          76 :   for (unsigned i=0; i < NumOperands; ++i) {
    2823         108 :     SDValue InOp = N->getOperand(i);
    2824          54 :     if (InputWidened)
    2825          54 :       InOp = GetWidenedVector(InOp);
    2826         418 :     for (unsigned j=0; j < NumInElts; ++j)
    2827         546 :       Ops[Idx++] = DAG.getNode(
    2828             :           ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    2829         728 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2830             :   }
    2831          22 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    2832         154 :   for (; Idx < WidenNumElts; ++Idx)
    2833         132 :     Ops[Idx] = UndefVal;
    2834          44 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    2835             : }
    2836             : 
    2837        2359 : SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
    2838        4718 :   EVT      VT = N->getValueType(0);
    2839        4718 :   EVT      WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2840        2359 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2841        4718 :   SDValue  InOp = N->getOperand(0);
    2842        4718 :   SDValue  Idx  = N->getOperand(1);
    2843        4718 :   SDLoc dl(N);
    2844             : 
    2845        7077 :   if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
    2846         318 :     InOp = GetWidenedVector(InOp);
    2847             : 
    2848        4718 :   EVT InVT = InOp.getValueType();
    2849             : 
    2850             :   // Check if we can just return the input vector after widening.
    2851        4718 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    2852        2359 :   if (IdxVal == 0 && InVT == WidenVT)
    2853        1836 :     return InOp;
    2854             : 
    2855             :   // Check if we can extract from the vector.
    2856         523 :   unsigned InNumElts = InVT.getVectorNumElements();
    2857         528 :   if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
    2858          10 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
    2859             : 
    2860             :   // We could try widening the input to the right length but for now, extract
    2861             :   // the original elements, fill the rest with undefs and build a vector.
    2862        1036 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    2863         518 :   EVT EltVT = VT.getVectorElementType();
    2864         518 :   unsigned NumElts = VT.getVectorNumElements();
    2865             :   unsigned i;
    2866        2126 :   for (i=0; i < NumElts; ++i)
    2867        3216 :     Ops[i] =
    2868        4824 :         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    2869             :                     DAG.getConstant(IdxVal + i, dl,
    2870        6432 :                                     TLI.getVectorIdxTy(DAG.getDataLayout())));
    2871             : 
    2872         518 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    2873        6454 :   for (; i < WidenNumElts; ++i)
    2874        5936 :     Ops[i] = UndefVal;
    2875        1036 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    2876             : }
    2877             : 
    2878          25 : SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
    2879          50 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2880          75 :   return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
    2881             :                      InOp.getValueType(), InOp,
    2882         150 :                      N->getOperand(1), N->getOperand(2));
    2883             : }
    2884             : 
    2885         563 : SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
    2886         563 :   LoadSDNode *LD = cast<LoadSDNode>(N);
    2887         563 :   ISD::LoadExtType ExtType = LD->getExtensionType();
    2888             : 
    2889         563 :   SDValue Result;
    2890        1126 :   SmallVector<SDValue, 16> LdChain;  // Chain for the series of load
    2891         563 :   if (ExtType != ISD::NON_EXTLOAD)
    2892           0 :     Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
    2893             :   else
    2894         563 :     Result = GenWidenVectorLoads(LdChain, LD);
    2895             : 
    2896             :   // If we generate a single load, we can use that for the chain.  Otherwise,
    2897             :   // build a factor node to remember the multiple loads are independent and
    2898             :   // chain to that.
    2899         563 :   SDValue NewChain;
    2900         563 :   if (LdChain.size() == 1)
    2901         348 :     NewChain = LdChain[0];
    2902             :   else
    2903        1290 :     NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
    2904             : 
    2905             :   // Modified the chain - switch anything that used the old chain to use
    2906             :   // the new one.
    2907         563 :   ReplaceValueWith(SDValue(N, 1), NewChain);
    2908             : 
    2909        1126 :   return Result;
    2910             : }
    2911             : 
    2912          10 : SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
    2913             : 
    2914          30 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
    2915          20 :   SDValue Mask = N->getMask();
    2916          20 :   EVT MaskVT = Mask.getValueType();
    2917          10 :   SDValue Src0 = GetWidenedVector(N->getSrc0());
    2918          10 :   ISD::LoadExtType ExtType = N->getExtensionType();
    2919          20 :   SDLoc dl(N);
    2920             : 
    2921          20 :   if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
    2922           0 :     Mask = GetWidenedVector(Mask);
    2923             :   else {
    2924          10 :     EVT BoolVT = getSetCCResultType(WidenVT);
    2925             : 
    2926             :     // We can't use ModifyToType() because we should fill the mask with
    2927             :     // zeroes
    2928          10 :     unsigned WidenNumElts = BoolVT.getVectorNumElements();
    2929          10 :     unsigned MaskNumElts = MaskVT.getVectorNumElements();
    2930             : 
    2931          10 :     unsigned NumConcat = WidenNumElts / MaskNumElts;
    2932          30 :     SmallVector<SDValue, 16> Ops(NumConcat);
    2933          10 :     SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
    2934          10 :     Ops[0] = Mask;
    2935          20 :     for (unsigned i = 1; i != NumConcat; ++i)
    2936          20 :       Ops[i] = ZeroVal;
    2937             : 
    2938          30 :     Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
    2939             :   }
    2940             : 
    2941          40 :   SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
    2942             :                                   Mask, Src0, N->getMemoryVT(),
    2943             :                                   N->getMemOperand(), ExtType,
    2944          30 :                                         N->isExpandingLoad());
    2945             :   // Legalize the chain result - switch anything that used the old chain to
    2946             :   // use the new one.
    2947          30 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    2948          20 :   return Res;
    2949             : }
    2950             : 
    2951          15 : SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
    2952             : 
    2953          45 :   EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2954          30 :   SDValue Mask = N->getMask();
    2955          30 :   SDValue Src0 = GetWidenedVector(N->getValue());
    2956          15 :   unsigned NumElts = WideVT.getVectorNumElements();
    2957          30 :   SDLoc dl(N);
    2958             : 
    2959             :   // The mask should be widened as well
    2960          15 :   Mask = WidenTargetBoolean(Mask, WideVT, true);
    2961             : 
    2962             :   // Widen the Index operand
    2963          30 :   SDValue Index = N->getIndex();
    2964          15 :   EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
    2965          45 :                                      Index.getValueType().getScalarType(),
    2966          30 :                                      NumElts);
    2967          15 :   Index = ModifyToType(Index, WideIndexVT);
    2968          45 :   SDValue Ops[] = { N->getChain(), Src0, Mask, N->getBasePtr(), Index };
    2969          45 :   SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
    2970             :                                     N->getMemoryVT(), dl, Ops,
    2971          45 :                                     N->getMemOperand());
    2972             : 
    2973             :   // Legalize the chain result - switch anything that used the old chain to
    2974             :   // use the new one.
    2975          45 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    2976          30 :   return Res;
    2977             : }
    2978             : 
    2979           4 : SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
    2980          12 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2981          12 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
    2982          16 :                      WidenVT, N->getOperand(0));
    2983             : }
    2984             : 
    2985             : // Return true if this is a node that could have two SETCCs as operands.
    2986             : static inline bool isLogicalMaskOp(unsigned Opcode) {
    2987        1431 :   switch (Opcode) {
    2988             :   case ISD::AND:
    2989             :   case ISD::OR:
    2990             :   case ISD::XOR:
    2991             :     return true;
    2992             :   }
    2993             :   return false;
    2994             : }
    2995             : 
    2996             : // This is used just for the assert in convertMask(). Check that this either
    2997             : // a SETCC or a previously handled SETCC by convertMask().
    2998             : #ifndef NDEBUG
    2999             : static inline bool isSETCCorConvertedSETCC(SDValue N) {
    3000             :   if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
    3001             :     N = N.getOperand(0);
    3002             :   else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
    3003             :     for (unsigned i = 1; i < N->getNumOperands(); ++i)
    3004             :       if (!N->getOperand(i)->isUndef())
    3005             :         return false;
    3006             :     N = N.getOperand(0);
    3007             :   }
    3008             : 
    3009             :   if (N.getOpcode() == ISD::TRUNCATE)
    3010             :     N = N.getOperand(0);
    3011             :   else if (N.getOpcode() == ISD::SIGN_EXTEND)
    3012             :     N = N.getOperand(0);
    3013             : 
    3014             :   if (isLogicalMaskOp(N.getOpcode()))
    3015             :     return isSETCCorConvertedSETCC(N.getOperand(0)) &&
    3016             :            isSETCCorConvertedSETCC(N.getOperand(1));
    3017             : 
    3018             :   return (N.getOpcode() == ISD::SETCC ||
    3019             :           ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
    3020             : }
    3021             : #endif
    3022             : 
    3023             : // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
    3024             : // to ToMaskVT if needed with vector extension or truncation.
    3025        1913 : SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
    3026             :                                       EVT ToMaskVT) {
    3027             :   // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
    3028             :   // FIXME: This code seems to be too restrictive, we might consider
    3029             :   // generalizing it or dropping it.
    3030             :   assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
    3031             : 
    3032             :   // Make a new Mask node, with a legal result VT.
    3033        3826 :   SmallVector<SDValue, 4> Ops;
    3034        9367 :   for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
    3035       11082 :     Ops.push_back(InMask->getOperand(i));
    3036       11478 :   SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
    3037             : 
    3038             :   // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
    3039             :   // extend or truncate is needed.
    3040        1913 :   LLVMContext &Ctx = *DAG.getContext();
    3041        1913 :   unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
    3042        1913 :   unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
    3043        1913 :   if (MaskScalarBits < ToMaskScalBits) {
    3044             :     EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
    3045         162 :                                  MaskVT.getVectorNumElements());
    3046         648 :     Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
    3047        1751 :   } else if (MaskScalarBits > ToMaskScalBits) {
    3048             :     EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
    3049         198 :                                    MaskVT.getVectorNumElements());
    3050         792 :     Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
    3051             :   }
    3052             : 
    3053             :   assert(Mask->getValueType(0).getScalarSizeInBits() ==
    3054             :              ToMaskVT.getScalarSizeInBits() &&
    3055             :          "Mask should have the right element size by now.");
    3056             : 
    3057             :   // Adjust Mask to the right number of elements.
    3058        3826 :   unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
    3059        1913 :   if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
    3060         120 :     MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
    3061         240 :     SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
    3062         180 :     Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
    3063         120 :                        ZeroIdx);
    3064        1853 :   } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
    3065         102 :     unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
    3066         204 :     EVT SubVT = Mask->getValueType(0);
    3067         306 :     SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
    3068         102 :     SubOps[0] = Mask;
    3069         510 :     Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
    3070             :   }
    3071             : 
    3072             :   assert((Mask->getValueType(0) == ToMaskVT) &&
    3073             :          "A mask of ToMaskVT should have been produced by now.");
    3074             : 
    3075        3826 :   return Mask;
    3076             : }
    3077             : 
    3078             : // Get the target mask VT, and widen if needed.
    3079        1713 : EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
    3080             :   assert(SetCC->getOpcode() == ISD::SETCC);
    3081        1713 :   LLVMContext &Ctx = *DAG.getContext();
    3082        5139 :   EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
    3083        3426 :   if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
    3084         312 :     MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
    3085        1713 :   return MaskVT;
    3086             : }
    3087             : 
    3088             : // This method tries to handle VSELECT and its mask by legalizing operands
    3089             : // (which may require widening) and if needed adjusting the mask vector type
    3090             : // to match that of the VSELECT. Without it, many cases end up with
    3091             : // scalarization of the SETCC, with many unnecessary instructions.
    3092        2584 : SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
    3093        2584 :   LLVMContext &Ctx = *DAG.getContext();
    3094        5168 :   SDValue Cond = N->getOperand(0);
    3095             : 
    3096        2584 :   if (N->getOpcode() != ISD::VSELECT)
    3097           0 :     return SDValue();
    3098             : 
    3099        6199 :   if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
    3100        1031 :     return SDValue();
    3101             : 
    3102             :   // If this is a splitted VSELECT that was previously already handled, do
    3103             :   // nothing.
    3104        4659 :   if (Cond->getValueType(0).getScalarSizeInBits() != 1)
    3105           1 :     return SDValue();
    3106             : 
    3107        3104 :   EVT VSelVT = N->getValueType(0);
    3108             :   // Only handle vector types which are a power of 2.
    3109        3104 :   if (!isPowerOf2_64(VSelVT.getSizeInBits()))
    3110          18 :     return SDValue();
    3111             : 
    3112             :   // Don't touch if this will be scalarized.
    3113        1534 :   EVT FinalVT = VSelVT;
    3114        4877 :   while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
    3115         603 :     FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
    3116             : 
    3117        1534 :   if (FinalVT.getVectorNumElements() == 1)
    3118          19 :     return SDValue();
    3119             : 
    3120             :   // If there is support for an i1 vector mask, don't touch.
    3121        3030 :   if (Cond.getOpcode() == ISD::SETCC) {
    3122        3945 :     EVT SetCCOpVT = Cond->getOperand(0).getValueType();
    3123        4268 :     while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
    3124        1092 :       SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
    3125        1315 :     EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
    3126        1315 :     if (SetCCResVT.getScalarSizeInBits() == 1)
    3127           2 :       return SDValue();
    3128             :   }
    3129             : 
    3130             :   // Get the VT and operands for VSELECT, and widen if needed.
    3131        3026 :   SDValue VSelOp1 = N->getOperand(1);
    3132        3026 :   SDValue VSelOp2 = N->getOperand(2);
    3133        3026 :   if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
    3134         264 :     VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
    3135         132 :     VSelOp1 = GetWidenedVector(VSelOp1);
    3136         132 :     VSelOp2 = GetWidenedVector(VSelOp2);
    3137             :   }
    3138             : 
    3139             :   // The mask of the VSELECT should have integer elements.
    3140        1513 :   EVT ToMaskVT = VSelVT;
    3141        1513 :   if (!ToMaskVT.getScalarType().isInteger())
    3142         241 :     ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
    3143             : 
    3144        1513 :   SDValue Mask;
    3145        3026 :   if (Cond->getOpcode() == ISD::SETCC) {
    3146        1313 :     EVT MaskVT = getSETCCWidenedResultTy(Cond);
    3147        1313 :     Mask = convertMask(Cond, MaskVT, ToMaskVT);
    3148         600 :   } else if (isLogicalMaskOp(Cond->getOpcode()) &&
    3149         600 :              Cond->getOperand(0).getOpcode() == ISD::SETCC &&
    3150         400 :              Cond->getOperand(1).getOpcode() == ISD::SETCC) {
    3151             :     // Cond is (AND/OR/XOR (SETCC, SETCC))
    3152         400 :     SDValue SETCC0 = Cond->getOperand(0);
    3153         400 :     SDValue SETCC1 = Cond->getOperand(1);
    3154         200 :     EVT VT0 = getSETCCWidenedResultTy(SETCC0);
    3155         200 :     EVT VT1 = getSETCCWidenedResultTy(SETCC1);
    3156         200 :     unsigned ScalarBits0 = VT0.getScalarSizeInBits();
    3157         200 :     unsigned ScalarBits1 = VT1.getScalarSizeInBits();
    3158         200 :     unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
    3159             :     EVT MaskVT;
    3160             :     // If the two SETCCs have different VTs, either extend/truncate one of
    3161             :     // them to the other "towards" ToMaskVT, or truncate one and extend the
    3162             :     // other to ToMaskVT.
    3163         200 :     if (ScalarBits0 != ScalarBits1) {
    3164         101 :       EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
    3165         101 :       EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
    3166         101 :       if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
    3167          41 :         MaskVT = WideVT;
    3168          60 :       else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
    3169          42 :         MaskVT = NarrowVT;
    3170             :       else
    3171          18 :         MaskVT = ToMaskVT;
    3172             :     } else
    3173             :       // If the two SETCCs have the same VT, don't change it.
    3174          99 :       MaskVT = VT0;
    3175             : 
    3176             :     // Make new SETCCs and logical nodes.
    3177         200 :     SETCC0 = convertMask(SETCC0, VT0, MaskVT);
    3178         200 :     SETCC1 = convertMask(SETCC1, VT1, MaskVT);
    3179        1000 :     Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
    3180             : 
    3181             :     // Convert the logical op for VSELECT if needed.
    3182         200 :     Mask = convertMask(Cond, MaskVT, ToMaskVT);
    3183             :   } else
    3184           0 :     return SDValue();
    3185             : 
    3186        4539 :   return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
    3187             : }
    3188             : 
    3189         174 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
    3190         522 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3191         174 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3192             : 
    3193         348 :   SDValue Cond1 = N->getOperand(0);
    3194         348 :   EVT CondVT = Cond1.getValueType();
    3195         174 :   if (CondVT.isVector()) {
    3196         163 :     if (SDValue Res = WidenVSELECTAndMask(N))
    3197         132 :       return Res;
    3198             : 
    3199          31 :     EVT CondEltVT = CondVT.getVectorElementType();
    3200          31 :     EVT CondWidenVT =  EVT::getVectorVT(*DAG.getContext(),
    3201          31 :                                         CondEltVT, WidenNumElts);
    3202          62 :     if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
    3203          23 :       Cond1 = GetWidenedVector(Cond1);
    3204             : 
    3205             :     // If we have to split the condition there is no point in widening the
    3206             :     // select. This would result in an cycle of widening the select ->
    3207             :     // widening the condition operand -> splitting the condition operand ->
    3208             :     // splitting the select -> widening the select. Instead split this select
    3209             :     // further and widen the resulting type.
    3210          62 :     if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
    3211           4 :       SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
    3212           4 :       SDValue Res = ModifyToType(SplitSelect, WidenVT);
    3213           4 :       return Res;
    3214             :     }
    3215             : 
    3216          54 :     if (Cond1.getValueType() != CondWidenVT)
    3217           4 :       Cond1 = ModifyToType(Cond1, CondWidenVT);
    3218             :   }
    3219             : 
    3220          76 :   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
    3221          76 :   SDValue InOp2 = GetWidenedVector(N->getOperand(2));
    3222             :   assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
    3223         114 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
    3224          76 :                      WidenVT, Cond1, InOp1, InOp2);
    3225             : }
    3226             : 
    3227           0 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
    3228           0 :   SDValue InOp1 = GetWidenedVector(N->getOperand(2));
    3229           0 :   SDValue InOp2 = GetWidenedVector(N->getOperand(3));
    3230           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
    3231           0 :                      InOp1.getValueType(), N->getOperand(0),
    3232           0 :                      N->getOperand(1), InOp1, InOp2, N->getOperand(4));
    3233             : }
    3234             : 
    3235          34 : SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
    3236             :   assert(N->getValueType(0).isVector() ==
    3237             :          N->getOperand(0).getValueType().isVector() &&
    3238             :          "Scalar/Vector type mismatch");
    3239         102 :   if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N);
    3240             : 
    3241           0 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3242           0 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    3243           0 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    3244           0 :   return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT,
    3245           0 :                      InOp1, InOp2, N->getOperand(2));
    3246             : }
    3247             : 
    3248         493 : SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
    3249        1479 :  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3250         493 :  return DAG.getUNDEF(WidenVT);
    3251             : }
    3252             : 
    3253          50 : SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
    3254         100 :   EVT VT = N->getValueType(0);
    3255         100 :   SDLoc dl(N);
    3256             : 
    3257         100 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3258          50 :   unsigned NumElts = VT.getVectorNumElements();
    3259          50 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3260             : 
    3261         100 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    3262         100 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    3263             : 
    3264             :   // Adjust mask based on new input vector length.
    3265         100 :   SmallVector<int, 16> NewMask;
    3266         237 :   for (unsigned i = 0; i != NumElts; ++i) {
    3267         374 :     int Idx = N->getMaskElt(i);
    3268         187 :     if (Idx < (int)NumElts)
    3269         126 :       NewMask.push_back(Idx);
    3270             :     else
    3271          61 :       NewMask.push_back(Idx - NumElts + WidenNumElts);
    3272             :   }
    3273         516 :   for (unsigned i = NumElts; i != WidenNumElts; ++i)
    3274         233 :     NewMask.push_back(-1);
    3275         150 :   return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
    3276             : }
    3277             : 
    3278          34 : SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
    3279             :   assert(N->getValueType(0).isVector() &&
    3280             :          N->getOperand(0).getValueType().isVector() &&
    3281             :          "Operands must be vectors");
    3282         102 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3283          34 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3284             : 
    3285          68 :   SDValue InOp1 = N->getOperand(0);
    3286          68 :   EVT InVT = InOp1.getValueType();
    3287             :   assert(InVT.isVector() && "can not widen non-vector type");
    3288          34 :   EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
    3289          68 :                                    InVT.getVectorElementType(), WidenNumElts);
    3290             : 
    3291             :   // The input and output types often differ here, and it could be that while
    3292             :   // we'd prefer to widen the result type, the input operands have been split.
    3293             :   // In this case, we also need to split the result of this node as well.
    3294          68 :   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
    3295           2 :     SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
    3296           2 :     SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
    3297           2 :     return Res;
    3298             :   }
    3299             : 
    3300          32 :   InOp1 = GetWidenedVector(InOp1);
    3301          64 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    3302             : 
    3303             :   // Assume that the input and output will be widen appropriately.  If not,
    3304             :   // we will have to unroll it at some point.
    3305             :   assert(InOp1.getValueType() == WidenInVT &&
    3306             :          InOp2.getValueType() == WidenInVT &&
    3307             :          "Input not widened to expected type!");
    3308             :   (void)WidenInVT;
    3309          96 :   return DAG.getNode(ISD::SETCC, SDLoc(N),
    3310          96 :                      WidenVT, InOp1, InOp2, N->getOperand(2));
    3311             : }
    3312             : 
    3313             : 
    3314             : //===----------------------------------------------------------------------===//
    3315             : // Widen Vector Operand
    3316             : //===----------------------------------------------------------------------===//
    3317        5387 : bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
    3318             :   DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
    3319             :         N->dump(&DAG);
    3320             :         dbgs() << "\n");
    3321        5387 :   SDValue Res = SDValue();
    3322             : 
    3323             :   // See if the target wants to custom widen this node.
    3324       16161 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    3325             :     return false;
    3326             : 
    3327       10668 :   switch (N->getOpcode()) {
    3328           0 :   default:
    3329             : #ifndef NDEBUG
    3330             :     dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
    3331             :     N->dump(&DAG);
    3332             :     dbgs() << "\n";
    3333             : #endif
    3334           0 :     llvm_unreachable("Do not know how to widen this operator's operand!");
    3335             : 
    3336          13 :   case ISD::BITCAST:            Res = WidenVecOp_BITCAST(N); break;
    3337         982 :   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
    3338          17 :   case ISD::EXTRACT_SUBVECTOR:  Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
    3339        2463 :   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
    3340         657 :   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
    3341           2 :   case ISD::MSTORE:             Res = WidenVecOp_MSTORE(N, OpNo); break;
    3342           5 :   case ISD::MSCATTER:           Res = WidenVecOp_MSCATTER(N, OpNo); break;
    3343         331 :   case ISD::SETCC:              Res = WidenVecOp_SETCC(N); break;
    3344           6 :   case ISD::FCOPYSIGN:          Res = WidenVecOp_FCOPYSIGN(N); break;
    3345             : 
    3346         767 :   case ISD::ANY_EXTEND:
    3347             :   case ISD::SIGN_EXTEND:
    3348             :   case ISD::ZERO_EXTEND:
    3349         767 :     Res = WidenVecOp_EXTEND(N);
    3350         767 :     break;
    3351             : 
    3352          94 :   case ISD::FP_EXTEND:
    3353             :   case ISD::FP_TO_SINT:
    3354             :   case ISD::FP_TO_UINT:
    3355             :   case ISD::SINT_TO_FP:
    3356             :   case ISD::UINT_TO_FP:
    3357             :   case ISD::TRUNCATE:
    3358          94 :     Res = WidenVecOp_Convert(N);
    3359          94 :     break;
    3360             :   }
    3361             : 
    3362             :   // If Res is null, the sub-method took care of registering the result.
    3363        5334 :   if (!Res.getNode()) return false;
    3364             : 
    3365             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    3366             :   // core about this.
    3367        5334 :   if (Res.getNode() == N)
    3368             :     return true;
    3369             : 
    3370             : 
    3371             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    3372             :          "Invalid operand expansion");
    3373             : 
    3374        5334 :   ReplaceValueWith(SDValue(N, 0), Res);
    3375        5334 :   return false;
    3376             : }
    3377             : 
    3378         767 : SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
    3379        1534 :   SDLoc DL(N);
    3380        1534 :   EVT VT = N->getValueType(0);
    3381             : 
    3382        1534 :   SDValue InOp = N->getOperand(0);
    3383             :   // If some legalization strategy other than widening is used on the operand,
    3384             :   // we can't safely assume that just extending the low lanes is the correct
    3385             :   // transformation.
    3386        2301 :   if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector)
    3387           0 :     return WidenVecOp_Convert(N);
    3388         767 :   InOp = GetWidenedVector(InOp);
    3389             :   assert(VT.getVectorNumElements() <
    3390             :              InOp.getValueType().getVectorNumElements() &&
    3391             :          "Input wasn't widened!");
    3392             : 
    3393             :   // We may need to further widen the operand until it has the same total
    3394             :   // vector size as the result.
    3395        1534 :   EVT InVT = InOp.getValueType();
    3396         767 :   if (InVT.getSizeInBits() != VT.getSizeInBits()) {
    3397           0 :     EVT InEltVT = InVT.getVectorElementType();
    3398           0 :     for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
    3399           0 :       EVT FixedVT = (MVT::SimpleValueType)i;
    3400           0 :       EVT FixedEltVT = FixedVT.getVectorElementType();
    3401           0 :       if (TLI.isTypeLegal(FixedVT) &&
    3402           0 :           FixedVT.getSizeInBits() == VT.getSizeInBits() &&
    3403           0 :           FixedEltVT == InEltVT) {
    3404             :         assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
    3405             :                "Not enough elements in the fixed type for the operand!");
    3406             :         assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
    3407             :                "We can't have the same type as we started with!");
    3408           0 :         if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
    3409           0 :           InOp = DAG.getNode(
    3410           0 :               ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
    3411           0 :               DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3412             :         else
    3413           0 :           InOp = DAG.getNode(
    3414             :               ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
    3415           0 :               DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3416           0 :         break;
    3417             :       }
    3418             :     }
    3419           0 :     InVT = InOp.getValueType();
    3420           0 :     if (InVT.getSizeInBits() != VT.getSizeInBits())
    3421             :       // We couldn't find a legal vector type that was a widening of the input
    3422             :       // and could be extended in-register to the result type, so we have to
    3423             :       // scalarize.
    3424           0 :       return WidenVecOp_Convert(N);
    3425             :   }
    3426             : 
    3427             :   // Use special DAG nodes to represent the operation of extending the
    3428             :   // low lanes.
    3429        1534 :   switch (N->getOpcode()) {
    3430           0 :   default:
    3431           0 :     llvm_unreachable("Extend legalization on on extend operation!");
    3432           1 :   case ISD::ANY_EXTEND:
    3433           1 :     return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
    3434         752 :   case ISD::SIGN_EXTEND:
    3435         752 :     return DAG.getSignExtendVectorInReg(InOp, DL, VT);
    3436          14 :   case ISD::ZERO_EXTEND:
    3437          14 :     return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
    3438             :   }
    3439             : }
    3440             : 
    3441           0 : SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
    3442             :   // The result (and first input) is legal, but the second input is illegal.
    3443             :   // We can't do much to fix that, so just unroll and let the extracts off of
    3444             :   // the second input be widened as needed later.
    3445           3 :   return DAG.UnrollVectorOp(N);
    3446             : }
    3447             : 
    3448          94 : SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
    3449             :   // Since the result is legal and the input is illegal, it is unlikely that we
    3450             :   // can fix the input to a legal type so unroll the convert into some scalar
    3451             :   // code and create a nasty build vector.
    3452         188 :   EVT VT = N->getValueType(0);
    3453          94 :   EVT EltVT = VT.getVectorElementType();
    3454         188 :   SDLoc dl(N);
    3455          94 :   unsigned NumElts = VT.getVectorNumElements();
    3456         188 :   SDValue InOp = N->getOperand(0);
    3457         282 :   if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
    3458          94 :     InOp = GetWidenedVector(InOp);
    3459         188 :   EVT InVT = InOp.getValueType();
    3460          94 :   EVT InEltVT = InVT.getVectorElementType();
    3461             : 
    3462         188 :   unsigned Opcode = N->getOpcode();
    3463         282 :   SmallVector<SDValue, 16> Ops(NumElts);
    3464         288 :   for (unsigned i=0; i < NumElts; ++i)
    3465         582 :     Ops[i] = DAG.getNode(
    3466             :         Opcode, dl, EltVT,
    3467         194 :         DAG.getNode(
    3468             :             ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
    3469         970 :             DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3470             : 
    3471         282 :   return DAG.getBuildVector(VT, dl, Ops);
    3472             : }
    3473             : 
    3474          13 : SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
    3475          26 :   EVT VT = N->getValueType(0);
    3476          26 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3477          26 :   EVT InWidenVT = InOp.getValueType();
    3478          26 :   SDLoc dl(N);
    3479             : 
    3480             :   // Check if we can convert between two legal vector types and extract.
    3481          13 :   unsigned InWidenSize = InWidenVT.getSizeInBits();
    3482          13 :   unsigned Size = VT.getSizeInBits();
    3483             :   // x86mmx is not an acceptable vector element type, so don't try.
    3484          39 :   if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
    3485          13 :     unsigned NewNumElts = InWidenSize / Size;
    3486          13 :     EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
    3487          26 :     if (TLI.isTypeLegal(NewVT)) {
    3488          26 :       SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
    3489          13 :       return DAG.getNode(
    3490             :           ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
    3491          39 :           DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3492             :     }
    3493             :   }
    3494             : 
    3495           0 :   return CreateStackStoreLoad(InOp, VT);
    3496             : }
    3497             : 
    3498         982 : SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
    3499             :   // If the input vector is not legal, it is likely that we will not find a
    3500             :   // legal vector of the same size. Replace the concatenate vector with a
    3501             :   // nasty build vector.
    3502        1964 :   EVT VT = N->getValueType(0);
    3503         982 :   EVT EltVT = VT.getVectorElementType();
    3504        1964 :   SDLoc dl(N);
    3505         982 :   unsigned NumElts = VT.getVectorNumElements();
    3506        2946 :   SmallVector<SDValue, 16> Ops(NumElts);
    3507             : 
    3508        2946 :   EVT InVT = N->getOperand(0).getValueType();
    3509         982 :   unsigned NumInElts = InVT.getVectorNumElements();
    3510             : 
    3511         982 :   unsigned Idx = 0;
    3512        1964 :   unsigned NumOperands = N->getNumOperands();
    3513        3456 :   for (unsigned i=0; i < NumOperands; ++i) {
    3514        4948 :     SDValue InOp = N->getOperand(i);
    3515        7422 :     if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
    3516        2474 :       InOp = GetWidenedVector(InOp);
    3517        8718 :     for (unsigned j=0; j < NumInElts; ++j)
    3518       18732 :       Ops[Idx++] = DAG.getNode(
    3519             :           ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    3520       24976 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3521             :   }
    3522        2946 :   return DAG.getBuildVector(VT, dl, Ops);
    3523             : }
    3524             : 
    3525          17 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
    3526          34 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3527          51 :   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
    3528          85 :                      N->getValueType(0), InOp, N->getOperand(1));
    3529             : }
    3530             : 
    3531        2463 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    3532        4926 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3533        7389 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
    3534       12315 :                      N->getValueType(0), InOp, N->getOperand(1));
    3535             : }
    3536             : 
    3537         657 : SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
    3538             :   // We have to widen the value, but we want only to store the original
    3539             :   // vector type.
    3540         657 :   StoreSDNode *ST = cast<StoreSDNode>(N);
    3541             : 
    3542        1314 :   SmallVector<SDValue, 16> StChain;
    3543         657 :   if (ST->isTruncatingStore())
    3544           0 :     GenWidenVectorTruncStores(StChain, ST);
    3545             :   else
    3546         657 :     GenWidenVectorStores(StChain, ST);
    3547             : 
    3548         657 :   if (StChain.size() == 1)
    3549          86 :     return StChain[0];
    3550             :   else
    3551        3426 :     return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
    3552             : }
    3553             : 
    3554           6 : SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
    3555           6 :   MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
    3556          12 :   SDValue Mask = MST->getMask();
    3557          12 :   EVT MaskVT = Mask.getValueType();
    3558           6 :   SDValue StVal = MST->getValue();
    3559             :   // Widen the value
    3560           6 :   SDValue WideVal = GetWidenedVector(StVal);
    3561          12 :   SDLoc dl(N);
    3562             : 
    3563          12 :   if (OpNo == 2 || getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
    3564           0 :     Mask = GetWidenedVector(Mask);
    3565             :   else {
    3566             :     // The mask should be widened as well.
    3567          12 :     EVT BoolVT = getSetCCResultType(WideVal.getValueType());
    3568             :     // We can't use ModifyToType() because we should fill the mask with
    3569             :     // zeroes.
    3570           6 :     unsigned WidenNumElts = BoolVT.getVectorNumElements();
    3571           6 :     unsigned MaskNumElts = MaskVT.getVectorNumElements();
    3572             : 
    3573           6 :     unsigned NumConcat = WidenNumElts / MaskNumElts;
    3574          18 :     SmallVector<SDValue, 16> Ops(NumConcat);
    3575           6 :     SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
    3576           6 :     Ops[0] = Mask;
    3577          12 :     for (unsigned i = 1; i != NumConcat; ++i)
    3578          12 :       Ops[i] = ZeroVal;
    3579             : 
    3580          18 :     Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
    3581             :   }
    3582             :   assert(Mask.getValueType().getVectorNumElements() ==
    3583             :          WideVal.getValueType().getVectorNumElements() &&
    3584             :          "Mask and data vectors should have the same number of elements");
    3585          24 :   return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
    3586             :                             Mask, MST->getMemoryVT(), MST->getMemOperand(),
    3587          18 :                             false, MST->isCompressingStore());
    3588             : }
    3589             : 
    3590           5 : SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
    3591             :   assert(OpNo == 1 && "Can widen only data operand of mscatter");
    3592           5 :   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
    3593          10 :   SDValue DataOp = MSC->getValue();
    3594          10 :   SDValue Mask = MSC->getMask();
    3595             : 
    3596             :   // Widen the value.
    3597           5 :   SDValue WideVal = GetWidenedVector(DataOp);
    3598          10 :   EVT WideVT = WideVal.getValueType();
    3599          10 :   unsigned NumElts = WideVal.getValueType().getVectorNumElements();
    3600          10 :   SDLoc dl(N);
    3601             : 
    3602             :   // The mask should be widened as well.
    3603           5 :   Mask = WidenTargetBoolean(Mask, WideVT, true);
    3604             : 
    3605             :   // Widen index.
    3606          10 :   SDValue Index = MSC->getIndex();
    3607           5 :   EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
    3608          15 :                                      Index.getValueType().getScalarType(),
    3609          10 :                                      NumElts);
    3610           5 :   Index = ModifyToType(Index, WideIndexVT);
    3611             : 
    3612          15 :   SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index};
    3613          15 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
    3614             :                               MSC->getMemoryVT(), dl, Ops,
    3615          20 :                               MSC->getMemOperand());
    3616             : }
    3617             : 
    3618         331 : SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
    3619         662 :   SDValue InOp0 = GetWidenedVector(N->getOperand(0));
    3620         662 :   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
    3621         662 :   SDLoc dl(N);
    3622             : 
    3623             :   // WARNING: In this code we widen the compare instruction with garbage.
    3624             :   // This garbage may contain denormal floats which may be slow. Is this a real
    3625             :   // concern ? Should we zero the unused lanes if this is a float compare ?
    3626             : 
    3627             :   // Get a new SETCC node to compare the newly widened operands.
    3628             :   // Only some of the compared elements are legal.
    3629         662 :   EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
    3630         993 :                                    InOp0.getValueType());
    3631         993 :   SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
    3632         993 :                      SVT, InOp0, InOp1, N->getOperand(2));
    3633             : 
    3634             :   // Extract the needed results from the result vector.
    3635         331 :   EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
    3636             :                                SVT.getVectorElementType(),
    3637         993 :                                N->getValueType(0).getVectorNumElements());
    3638         331 :   SDValue CC = DAG.getNode(
    3639             :       ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
    3640         993 :       DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3641             : 
    3642         993 :   return PromoteTargetBoolean(CC, N->getValueType(0));
    3643             : }
    3644             : 
    3645             : 
    3646             : //===----------------------------------------------------------------------===//
    3647             : // Vector Widening Utilities
    3648             : //===----------------------------------------------------------------------===//
    3649             : 
    3650             : // Utility function to find the type to chop up a widen vector for load/store
    3651             : //  TLI:       Target lowering used to determine legal types.
    3652             : //  Width:     Width left need to load/store.
    3653             : //  WidenVT:   The widen vector type to load to/store from
    3654             : //  Align:     If 0, don't allow use of a wider type
    3655             : //  WidenEx:   If Align is not 0, the amount additional we can load/store from.
    3656             : 
    3657        1843 : static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
    3658             :                        unsigned Width, EVT WidenVT,
    3659             :                        unsigned Align = 0, unsigned WidenEx = 0) {
    3660        1843 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    3661        1843 :   unsigned WidenWidth = WidenVT.getSizeInBits();
    3662        1843 :   unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
    3663        1843 :   unsigned AlignInBits = Align*8;
    3664             : 
    3665             :   // If we have one element to load/store, return it.
    3666        1843 :   EVT RetVT = WidenEltVT;
    3667        1843 :   if (Width == WidenEltWidth)
    3668         457 :     return RetVT;
    3669             : 
    3670             :   // See if there is larger legal integer than the element type to load/store.
    3671             :   unsigned VT;
    3672        2064 :   for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
    3673        3450 :        VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
    3674        6900 :     EVT MemVT((MVT::SimpleValueType) VT);
    3675        3450 :     unsigned MemVTWidth = MemVT.getSizeInBits();
    3676        3450 :     if (MemVT.getSizeInBits() <= WidenEltWidth)
    3677             :       break;
    3678        6334 :     auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
    3679             :     if ((Action == TargetLowering::TypeLegal ||
    3680        1509 :          Action == TargetLowering::TypePromoteInteger) &&
    3681        1509 :         (WidenWidth % MemVTWidth) == 0 &&
    3682        5735 :         isPowerOf2_32(WidenWidth / MemVTWidth) &&
    3683         289 :         (MemVTWidth <= Width ||
    3684         397 :          (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
    3685        1103 :       RetVT = MemVT;
    3686        1103 :       break;
    3687             :     }
    3688             :   }
    3689             : 
    3690             :   // See if there is a larger vector type to load/store that has the same vector
    3691             :   // element type and is evenly divisible with the WidenVT.
    3692      117229 :   for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
    3693      118615 :        VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
    3694      235322 :     EVT MemVT = (MVT::SimpleValueType) VT;
    3695      117661 :     unsigned MemVTWidth = MemVT.getSizeInBits();
    3696      130843 :     if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
    3697        2247 :         (WidenWidth % MemVTWidth) == 0 &&
    3698        2986 :         isPowerOf2_32(WidenWidth / MemVTWidth) &&
    3699         948 :         (MemVTWidth <= Width ||
    3700        1070 :          (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
    3701         667 :       if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
    3702         432 :         return MemVT;
    3703             :     }
    3704             :   }
    3705             : 
    3706         954 :   return RetVT;
    3707             : }
    3708             : 
    3709             : // Builds a vector type from scalar loads
    3710             : //  VecTy: Resulting Vector type
    3711             : //  LDOps: Load operators to build a vector type
    3712             : //  [Start,End) the list of loads to use.
    3713         170 : static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
    3714             :                                      SmallVectorImpl<SDValue> &LdOps,
    3715             :                                      unsigned Start, unsigned End) {
    3716         170 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    3717         680 :   SDLoc dl(LdOps[Start]);
    3718         510 :   EVT LdTy = LdOps[Start].getValueType();
    3719         170 :   unsigned Width = VecTy.getSizeInBits();
    3720         170 :   unsigned NumElts = Width / LdTy.getSizeInBits();
    3721         170 :   EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
    3722             : 
    3723         170 :   unsigned Idx = 1;
    3724         340 :   SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
    3725             : 
    3726         404 :   for (unsigned i = Start + 1; i != End; ++i) {
    3727         702 :     EVT NewLdTy = LdOps[i].getValueType();
    3728         258 :     if (NewLdTy != LdTy) {
    3729          74 :       NumElts = Width / NewLdTy.getSizeInBits();
    3730          74 :       NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
    3731          74 :       VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
    3732             :       // Readjust position and vector position based on new load type.
    3733          74 :       Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
    3734          74 :       LdTy = NewLdTy;
    3735             :     }
    3736         234 :     VecOp = DAG.getNode(
    3737         468 :         ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
    3738         936 :         DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3739             :   }
    3740         340 :   return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
    3741             : }
    3742             : 
    3743         563 : SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
    3744             :                                               LoadSDNode *LD) {
    3745             :   // The strategy assumes that we can efficiently load power-of-two widths.
    3746             :   // The routine chops the vector into the largest vector loads with the same
    3747             :   // element type or scalar loads and then recombines it to the widen vector
    3748             :   // type.
    3749        1689 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
    3750         563 :   unsigned WidenWidth = WidenVT.getSizeInBits();
    3751         563 :   EVT LdVT    = LD->getMemoryVT();
    3752        1126 :   SDLoc dl(LD);
    3753             :   assert(LdVT.isVector() && WidenVT.isVector());
    3754             :   assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
    3755             : 
    3756             :   // Load information
    3757        1126 :   SDValue Chain = LD->getChain();
    3758         563 :   SDValue BasePtr = LD->getBasePtr();
    3759        1126 :   unsigned Align = LD->getAlignment();
    3760         563 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    3761         563 :   AAMDNodes AAInfo = LD->getAAInfo();
    3762             : 
    3763         563 :   int LdWidth = LdVT.getSizeInBits();
    3764         563 :   int WidthDiff = WidenWidth - LdWidth;
    3765        1126 :   unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
    3766             : 
    3767             :   // Find the vector type that can load from.
    3768         563 :   EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
    3769         563 :   int NewVTWidth = NewVT.getSizeInBits();
    3770        1126 :   SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
    3771        1126 :                              Align, MMOFlags, AAInfo);
    3772        1126 :   LdChain.push_back(LdOp.getValue(1));
    3773             : 
    3774             :   // Check if we can load the element with one instruction.
    3775         563 :   if (LdWidth <= NewVTWidth) {
    3776         348 :     if (!NewVT.isVector()) {
    3777         249 :       unsigned NumElts = WidenWidth / NewVTWidth;
    3778         249 :       EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
    3779         498 :       SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
    3780         498 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
    3781             :     }
    3782          99 :     if (NewVT == WidenVT)
    3783          99 :       return LdOp;
    3784             : 
    3785             :     assert(WidenWidth % NewVTWidth == 0);
    3786           0 :     unsigned NumConcat = WidenWidth / NewVTWidth;
    3787           0 :     SmallVector<SDValue, 16> ConcatOps(NumConcat);
    3788           0 :     SDValue UndefVal = DAG.getUNDEF(NewVT);
    3789           0 :     ConcatOps[0] = LdOp;
    3790           0 :     for (unsigned i = 1; i != NumConcat; ++i)
    3791           0 :       ConcatOps[i] = UndefVal;
    3792           0 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
    3793             :   }
    3794             : 
    3795             :   // Load vector by using multiple loads from largest vector to scalar.
    3796         215 :   SmallVector<SDValue, 16> LdOps;
    3797         215 :   LdOps.push_back(LdOp);
    3798             : 
    3799         215 :   LdWidth -= NewVTWidth;
    3800         215 :   unsigned Offset = 0;
    3801             : 
    3802        1625 :   while (LdWidth > 0) {
    3803         705 :     unsigned Increment = NewVTWidth / 8;
    3804         705 :     Offset += Increment;
    3805        1410 :     BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
    3806        2115 :                           DAG.getConstant(Increment, dl, BasePtr.getValueType()));
    3807             : 
    3808         705 :     SDValue L;
    3809         705 :     if (LdWidth < NewVTWidth) {
    3810             :       // The current type we are using is too large. Find a better size.
    3811         126 :       NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
    3812         126 :       NewVTWidth = NewVT.getSizeInBits();
    3813         252 :       L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
    3814         126 :                       LD->getPointerInfo().getWithOffset(Offset),
    3815         504 :                       MinAlign(Align, Increment), MMOFlags, AAInfo);
    3816         252 :       LdChain.push_back(L.getValue(1));
    3817         378 :       if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
    3818             :         // Later code assumes the vector loads produced will be mergeable, so we
    3819             :         // must pad the final entry up to the previous width. Scalars are
    3820             :         // combined separately.
    3821          54 :         SmallVector<SDValue, 16> Loads;
    3822          27 :         Loads.push_back(L);
    3823          54 :         unsigned size = L->getValueSizeInBits(0);
    3824          69 :         while (size < LdOp->getValueSizeInBits(0)) {
    3825          10 :           Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
    3826          10 :           size += L->getValueSizeInBits(0);
    3827             :         }
    3828          81 :         L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
    3829             :       }
    3830             :     } else {
    3831        1158 :       L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
    3832         579 :                       LD->getPointerInfo().getWithOffset(Offset),
    3833        2316 :                       MinAlign(Align, Increment), MMOFlags, AAInfo);
    3834        1158 :       LdChain.push_back(L.getValue(1));
    3835             :     }
    3836             : 
    3837         705 :     LdOps.push_back(L);
    3838             : 
    3839             : 
    3840         705 :     LdWidth -= NewVTWidth;
    3841             :   }
    3842             : 
    3843             :   // Build the vector from the load operations.
    3844         215 :   unsigned End = LdOps.size();
    3845         645 :   if (!LdOps[0].getValueType().isVector())
    3846             :     // All the loads are scalar loads.
    3847         146 :     return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
    3848             : 
    3849             :   // If the load contains vectors, build the vector using concat vector.
    3850             :   // All of the vectors used to load are power-of-2, and the scalar loads can be
    3851             :   // combined to make a power-of-2 vector.
    3852         138 :   SmallVector<SDValue, 16> ConcatOps(End);
    3853          69 :   int i = End - 1;
    3854          69 :   int Idx = End;
    3855         207 :   EVT LdTy = LdOps[i].getValueType();
    3856             :   // First, combine the scalar loads to a vector.
    3857          69 :   if (!LdTy.isVector())  {
    3858          24 :     for (--i; i >= 0; --i) {
    3859          72 :       LdTy = LdOps[i].getValueType();
    3860          24 :       if (LdTy.isVector())
    3861             :         break;
    3862             :     }
    3863          48 :     ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
    3864             :   }
    3865         207 :   ConcatOps[--Idx] = LdOps[i];
    3866         516 :   for (--i; i >= 0; --i) {
    3867        1341 :     EVT NewLdTy = LdOps[i].getValueType();
    3868         447 :     if (NewLdTy != LdTy) {
    3869             :       // Create a larger vector.
    3870           3 :       ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
    3871           4 :                                      makeArrayRef(&ConcatOps[Idx], End - Idx));
    3872           1 :       Idx = End - 1;
    3873           1 :       LdTy = NewLdTy;
    3874             :     }
    3875        1341 :     ConcatOps[--Idx] = LdOps[i];
    3876             :   }
    3877             : 
    3878          69 :   if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
    3879          39 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
    3880         117 :                        makeArrayRef(&ConcatOps[Idx], End - Idx));
    3881             : 
    3882             :   // We need to fill the rest with undefs to build the vector.
    3883          30 :   unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
    3884          60 :   SmallVector<SDValue, 16> WidenOps(NumOps);
    3885          30 :   SDValue UndefVal = DAG.getUNDEF(LdTy);
    3886             :   {
    3887          30 :     unsigned i = 0;
    3888         952 :     for (; i != End-Idx; ++i)
    3889        1383 :       WidenOps[i] = ConcatOps[Idx+i];
    3890         412 :     for (; i != NumOps; ++i)
    3891         382 :       WidenOps[i] = UndefVal;
    3892             :   }
    3893          90 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
    3894             : }
    3895             : 
    3896             : SDValue
    3897           0 : DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
    3898             :                                          LoadSDNode *LD,
    3899             :                                          ISD::LoadExtType ExtType) {
    3900             :   // For extension loads, it may not be more efficient to chop up the vector
    3901             :   // and then extend it. Instead, we unroll the load and build a new vector.
    3902           0 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
    3903           0 :   EVT LdVT    = LD->getMemoryVT();
    3904           0 :   SDLoc dl(LD);
    3905             :   assert(LdVT.isVector() && WidenVT.isVector());
    3906             : 
    3907             :   // Load information
    3908           0 :   SDValue Chain = LD->getChain();
    3909           0 :   SDValue BasePtr = LD->getBasePtr();
    3910           0 :   unsigned Align = LD->getAlignment();
    3911           0 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    3912           0 :   AAMDNodes AAInfo = LD->getAAInfo();
    3913             : 
    3914           0 :   EVT EltVT = WidenVT.getVectorElementType();
    3915           0 :   EVT LdEltVT = LdVT.getVectorElementType();
    3916           0 :   unsigned NumElts = LdVT.getVectorNumElements();
    3917             : 
    3918             :   // Load each element and widen.
    3919           0 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3920           0 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    3921           0 :   unsigned Increment = LdEltVT.getSizeInBits() / 8;
    3922           0 :   Ops[0] =
    3923           0 :       DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
    3924           0 :                      LdEltVT, Align, MMOFlags, AAInfo);
    3925           0 :   LdChain.push_back(Ops[0].getValue(1));
    3926           0 :   unsigned i = 0, Offset = Increment;
    3927           0 :   for (i=1; i < NumElts; ++i, Offset += Increment) {
    3928           0 :     SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
    3929             :                                      BasePtr,
    3930             :                                      DAG.getConstant(Offset, dl,
    3931           0 :                                                      BasePtr.getValueType()));
    3932           0 :     Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
    3933           0 :                             LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
    3934           0 :                             Align, MMOFlags, AAInfo);
    3935           0 :     LdChain.push_back(Ops[i].getValue(1));
    3936             :   }
    3937             : 
    3938             :   // Fill the rest with undefs.
    3939           0 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    3940           0 :   for (; i != WidenNumElts; ++i)
    3941           0 :     Ops[i] = UndefVal;
    3942             : 
    3943           0 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    3944             : }
    3945             : 
    3946         657 : void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
    3947             :                                             StoreSDNode *ST) {
    3948             :   // The strategy assumes that we can efficiently store power-of-two widths.
    3949             :   // The routine chops the vector into the largest vector stores with the same
    3950             :   // element type or scalar stores.
    3951        1314 :   SDValue  Chain = ST->getChain();
    3952         657 :   SDValue  BasePtr = ST->getBasePtr();
    3953        1314 :   unsigned Align = ST->getAlignment();
    3954         657 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
    3955        1314 :   AAMDNodes AAInfo = ST->getAAInfo();
    3956         657 :   SDValue  ValOp = GetWidenedVector(ST->getValue());
    3957        1314 :   SDLoc dl(ST);
    3958             : 
    3959         657 :   EVT StVT = ST->getMemoryVT();
    3960         657 :   unsigned StWidth = StVT.getSizeInBits();
    3961        1314 :   EVT ValVT = ValOp.getValueType();
    3962         657 :   unsigned ValWidth = ValVT.getSizeInBits();
    3963         657 :   EVT ValEltVT = ValVT.getVectorElementType();
    3964         657 :   unsigned ValEltWidth = ValEltVT.getSizeInBits();
    3965             :   assert(StVT.getVectorElementType() == ValEltVT);
    3966             : 
    3967         657 :   int Idx = 0;          // current index to store
    3968         657 :   unsigned Offset = 0;  // offset from base to store
    3969        2965 :   while (StWidth != 0) {
    3970             :     // Find the largest vector type we can store with.
    3971        1154 :     EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
    3972        1154 :     unsigned NewVTWidth = NewVT.getSizeInBits();
    3973        1154 :     unsigned Increment = NewVTWidth / 8;
    3974        1154 :     if (NewVT.isVector()) {
    3975         236 :       unsigned NumVTElts = NewVT.getVectorNumElements();
    3976             :       do {
    3977         902 :         SDValue EOp = DAG.getNode(
    3978             :             ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
    3979        2706 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3980        1804 :         StChain.push_back(DAG.getStore(
    3981         902 :             Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
    3982        3608 :             MinAlign(Align, Offset), MMOFlags, AAInfo));
    3983         902 :         StWidth -= NewVTWidth;
    3984         902 :         Offset += Increment;
    3985         902 :         Idx += NumVTElts;
    3986        1804 :         BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
    3987             :                               DAG.getConstant(Increment, dl,
    3988        2706 :                                               BasePtr.getValueType()));
    3989         902 :       } while (StWidth != 0 && StWidth >= NewVTWidth);
    3990             :     } else {
    3991             :       // Cast the vector to the scalar type we can store.
    3992         918 :       unsigned NumElts = ValWidth / NewVTWidth;
    3993         918 :       EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
    3994        1836 :       SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
    3995             :       // Readjust index position based on new vector type.
    3996         918 :       Idx = Idx * ValEltWidth / NewVTWidth;
    3997             :       do {
    3998        1105 :         SDValue EOp = DAG.getNode(
    3999             :             ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
    4000        1105 :             DAG.getConstant(Idx++, dl,
    4001        4420 :                             TLI.getVectorIdxTy(DAG.getDataLayout())));
    4002        2210 :         StChain.push_back(DAG.getStore(
    4003        1105 :             Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
    4004        4420 :             MinAlign(Align, Offset), MMOFlags, AAInfo));
    4005        1105 :         StWidth -= NewVTWidth;
    4006        1105 :         Offset += Increment;
    4007        2210 :         BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
    4008             :                               DAG.getConstant(Increment, dl,
    4009        3315 :                                               BasePtr.getValueType()));
    4010        1105 :       } while (StWidth != 0 && StWidth >= NewVTWidth);
    4011             :       // Restore index back to be relative to the original widen element type.
    4012         918 :       Idx = Idx * NewVTWidth / ValEltWidth;
    4013             :     }
    4014             :   }
    4015         657 : }
    4016             : 
    4017             : void
    4018           0 : DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
    4019             :                                             StoreSDNode *ST) {
    4020             :   // For extension loads, it may not be more efficient to truncate the vector
    4021             :   // and then store it. Instead, we extract each element and then store it.
    4022           0 :   SDValue Chain = ST->getChain();
    4023           0 :   SDValue BasePtr = ST->getBasePtr();
    4024           0 :   unsigned Align = ST->getAlignment();
    4025           0 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
    4026           0 :   AAMDNodes AAInfo = ST->getAAInfo();
    4027           0 :   SDValue ValOp = GetWidenedVector(ST->getValue());
    4028           0 :   SDLoc dl(ST);
    4029             : 
    4030           0 :   EVT StVT = ST->getMemoryVT();
    4031           0 :   EVT ValVT = ValOp.getValueType();
    4032             : 
    4033             :   // It must be true that the wide vector type is bigger than where we need to
    4034             :   // store.
    4035             :   assert(StVT.isVector() && ValOp.getValueType().isVector());
    4036             :   assert(StVT.bitsLT(ValOp.getValueType()));
    4037             : 
    4038             :   // For truncating stores, we can not play the tricks of chopping legal vector
    4039             :   // types and bitcast it to the right type. Instead, we unroll the store.
    4040           0 :   EVT StEltVT  = StVT.getVectorElementType();
    4041           0 :   EVT ValEltVT = ValVT.getVectorElementType();
    4042           0 :   unsigned Increment = ValEltVT.getSizeInBits() / 8;
    4043           0 :   unsigned NumElts = StVT.getVectorNumElements();
    4044           0 :   SDValue EOp = DAG.getNode(
    4045             :       ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
    4046           0 :       DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4047           0 :   StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
    4048           0 :                                       ST->getPointerInfo(), StEltVT, Align,
    4049           0 :                                       MMOFlags, AAInfo));
    4050           0 :   unsigned Offset = Increment;
    4051           0 :   for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
    4052           0 :     SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
    4053             :                                      BasePtr,
    4054             :                                      DAG.getConstant(Offset, dl,
    4055           0 :                                                      BasePtr.getValueType()));
    4056           0 :     SDValue EOp = DAG.getNode(
    4057             :         ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
    4058           0 :         DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4059           0 :     StChain.push_back(DAG.getTruncStore(
    4060           0 :         Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
    4061           0 :         StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
    4062             :   }
    4063           0 : }
    4064             : 
    4065             : /// Modifies a vector input (widen or narrows) to a vector of NVT.  The
    4066             : /// input vector must have the same element type as NVT.
    4067             : /// FillWithZeroes specifies that the vector should be widened with zeroes.
    4068          50 : SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
    4069             :                                        bool FillWithZeroes) {
    4070             :   // Note that InOp might have been widened so it might already have
    4071             :   // the right width or it might need be narrowed.
    4072         100 :   EVT InVT = InOp.getValueType();
    4073             :   assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
    4074             :          "input and widen element type must match");
    4075         100 :   SDLoc dl(InOp);
    4076             : 
    4077             :   // Check if InOp already has the right width.
    4078          50 :   if (InVT == NVT)
    4079           0 :     return InOp;
    4080             : 
    4081          50 :   unsigned InNumElts = InVT.getVectorNumElements();
    4082          50 :   unsigned WidenNumElts = NVT.getVectorNumElements();
    4083          50 :   if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
    4084          50 :     unsigned NumConcat = WidenNumElts / InNumElts;
    4085         150 :     SmallVector<SDValue, 16> Ops(NumConcat);
    4086          20 :     SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
    4087          70 :       DAG.getUNDEF(InVT);
    4088          50 :     Ops[0] = InOp;
    4089         100 :     for (unsigned i = 1; i != NumConcat; ++i)
    4090         100 :       Ops[i] = FillVal;
    4091             : 
    4092         150 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
    4093             :   }
    4094             : 
    4095           0 :   if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
    4096           0 :     return DAG.getNode(
    4097             :         ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
    4098           0 :         DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4099             : 
    4100             :   // Fall back to extract and build.
    4101           0 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    4102           0 :   EVT EltVT = NVT.getVectorElementType();
    4103           0 :   unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
    4104             :   unsigned Idx;
    4105           0 :   for (Idx = 0; Idx < MinNumElts; ++Idx)
    4106           0 :     Ops[Idx] = DAG.getNode(
    4107             :         ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    4108           0 :         DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4109             : 
    4110           0 :   SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
    4111           0 :     DAG.getUNDEF(EltVT);
    4112           0 :   for ( ; Idx < WidenNumElts; ++Idx)
    4113           0 :     Ops[Idx] = FillVal;
    4114           0 :   return DAG.getBuildVector(NVT, dl, Ops);
    4115             : }

Generated by: LCOV version 1.13