LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeVectorTypes.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1976 2211 89.4 %
Date: 2018-09-23 13:06:45 Functions: 112 121 92.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file performs vector type splitting and scalarization for LegalizeTypes.
      11             : // Scalarization is the act of changing a computation in an illegal one-element
      12             : // vector type to be a computation in its scalar element type.  For example,
      13             : // implementing <1 x f32> arithmetic in a scalar f32 register.  This is needed
      14             : // as a base case when scalarizing vector arithmetic like <4 x f32>, which
      15             : // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
      16             : // types.
      17             : // Splitting is the act of changing a computation in an invalid vector type to
      18             : // be a computation in two vectors of half the size.  For example, implementing
      19             : // <128 x f32> operations in terms of two <64 x f32> operations.
      20             : //
      21             : //===----------------------------------------------------------------------===//
      22             : 
      23             : #include "LegalizeTypes.h"
      24             : #include "llvm/IR/DataLayout.h"
      25             : #include "llvm/Support/ErrorHandling.h"
      26             : #include "llvm/Support/raw_ostream.h"
      27             : using namespace llvm;
      28             : 
      29             : #define DEBUG_TYPE "legalize-types"
      30             : 
      31             : //===----------------------------------------------------------------------===//
      32             : //  Result Vector Scalarization: <1 x ty> -> ty.
      33             : //===----------------------------------------------------------------------===//
      34             : 
      35       62156 : void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
      36             :   LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
      37             :              dbgs() << "\n");
      38       62156 :   SDValue R = SDValue();
      39             : 
      40      124312 :   switch (N->getOpcode()) {
      41           0 :   default:
      42             : #ifndef NDEBUG
      43             :     dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
      44             :     N->dump(&DAG);
      45             :     dbgs() << "\n";
      46             : #endif
      47           0 :     report_fatal_error("Do not know how to scalarize the result of this "
      48             :                        "operator!\n");
      49             : 
      50           0 :   case ISD::MERGE_VALUES:      R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
      51       17024 :   case ISD::BITCAST:           R = ScalarizeVecRes_BITCAST(N); break;
      52        9285 :   case ISD::BUILD_VECTOR:      R = ScalarizeVecRes_BUILD_VECTOR(N); break;
      53        5194 :   case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
      54         754 :   case ISD::FP_ROUND:          R = ScalarizeVecRes_FP_ROUND(N); break;
      55           0 :   case ISD::FP_ROUND_INREG:    R = ScalarizeVecRes_InregOp(N); break;
      56           0 :   case ISD::FPOWI:             R = ScalarizeVecRes_FPOWI(N); break;
      57         148 :   case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
      58        5376 :   case ISD::LOAD:           R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
      59         151 :   case ISD::SCALAR_TO_VECTOR:  R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
      60          21 :   case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
      61         357 :   case ISD::VSELECT:           R = ScalarizeVecRes_VSELECT(N); break;
      62         209 :   case ISD::SELECT:            R = ScalarizeVecRes_SELECT(N); break;
      63           0 :   case ISD::SELECT_CC:         R = ScalarizeVecRes_SELECT_CC(N); break;
      64        1140 :   case ISD::SETCC:             R = ScalarizeVecRes_SETCC(N); break;
      65         133 :   case ISD::UNDEF:             R = ScalarizeVecRes_UNDEF(N); break;
      66           0 :   case ISD::VECTOR_SHUFFLE:    R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
      67          20 :   case ISD::ANY_EXTEND_VECTOR_INREG:
      68             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
      69             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
      70          20 :     R = ScalarizeVecRes_VecInregOp(N);
      71          20 :     break;
      72       17214 :   case ISD::ANY_EXTEND:
      73             :   case ISD::BITREVERSE:
      74             :   case ISD::BSWAP:
      75             :   case ISD::CTLZ:
      76             :   case ISD::CTLZ_ZERO_UNDEF:
      77             :   case ISD::CTPOP:
      78             :   case ISD::CTTZ:
      79             :   case ISD::CTTZ_ZERO_UNDEF:
      80             :   case ISD::FABS:
      81             :   case ISD::FCEIL:
      82             :   case ISD::FCOS:
      83             :   case ISD::FEXP:
      84             :   case ISD::FEXP2:
      85             :   case ISD::FFLOOR:
      86             :   case ISD::FLOG:
      87             :   case ISD::FLOG10:
      88             :   case ISD::FLOG2:
      89             :   case ISD::FNEARBYINT:
      90             :   case ISD::FNEG:
      91             :   case ISD::FP_EXTEND:
      92             :   case ISD::FP_TO_SINT:
      93             :   case ISD::FP_TO_UINT:
      94             :   case ISD::FRINT:
      95             :   case ISD::FROUND:
      96             :   case ISD::FSIN:
      97             :   case ISD::FSQRT:
      98             :   case ISD::FTRUNC:
      99             :   case ISD::SIGN_EXTEND:
     100             :   case ISD::SINT_TO_FP:
     101             :   case ISD::TRUNCATE:
     102             :   case ISD::UINT_TO_FP:
     103             :   case ISD::ZERO_EXTEND:
     104             :   case ISD::FCANONICALIZE:
     105       17214 :     R = ScalarizeVecRes_UnaryOp(N);
     106       17214 :     break;
     107             : 
     108        5080 :   case ISD::ADD:
     109             :   case ISD::AND:
     110             :   case ISD::FADD:
     111             :   case ISD::FCOPYSIGN:
     112             :   case ISD::FDIV:
     113             :   case ISD::FMUL:
     114             :   case ISD::FMINNUM:
     115             :   case ISD::FMAXNUM:
     116             :   case ISD::FMINNAN:
     117             :   case ISD::FMAXNAN:
     118             :   case ISD::SMIN:
     119             :   case ISD::SMAX:
     120             :   case ISD::UMIN:
     121             :   case ISD::UMAX:
     122             : 
     123             :   case ISD::FPOW:
     124             :   case ISD::FREM:
     125             :   case ISD::FSUB:
     126             :   case ISD::MUL:
     127             :   case ISD::OR:
     128             :   case ISD::SDIV:
     129             :   case ISD::SREM:
     130             :   case ISD::SUB:
     131             :   case ISD::UDIV:
     132             :   case ISD::UREM:
     133             :   case ISD::XOR:
     134             :   case ISD::SHL:
     135             :   case ISD::SRA:
     136             :   case ISD::SRL:
     137        5080 :     R = ScalarizeVecRes_BinOp(N);
     138        5080 :     break;
     139          14 :   case ISD::FMA:
     140          14 :     R = ScalarizeVecRes_TernaryOp(N);
     141          14 :     break;
     142          36 :   case ISD::STRICT_FADD:
     143             :   case ISD::STRICT_FSUB:
     144             :   case ISD::STRICT_FMUL:
     145             :   case ISD::STRICT_FDIV:
     146             :   case ISD::STRICT_FREM:
     147             :   case ISD::STRICT_FSQRT:
     148             :   case ISD::STRICT_FMA:
     149             :   case ISD::STRICT_FPOW:
     150             :   case ISD::STRICT_FPOWI:
     151             :   case ISD::STRICT_FSIN:
     152             :   case ISD::STRICT_FCOS:
     153             :   case ISD::STRICT_FEXP:
     154             :   case ISD::STRICT_FEXP2:
     155             :   case ISD::STRICT_FLOG:
     156             :   case ISD::STRICT_FLOG10:
     157             :   case ISD::STRICT_FLOG2:
     158             :   case ISD::STRICT_FRINT:
     159             :   case ISD::STRICT_FNEARBYINT:
     160          36 :     R = ScalarizeVecRes_StrictFPOp(N);
     161          36 :     break;
     162             :   }
     163             : 
     164             :   // If R is null, the sub-method took care of registering the result.
     165       62156 :   if (R.getNode())
     166       62156 :     SetScalarizedVector(SDValue(N, ResNo), R);
     167       62156 : }
     168             : 
     169        5080 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
     170       10160 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     171       10160 :   SDValue RHS = GetScalarizedVector(N->getOperand(1));
     172        5080 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     173        5080 :                      LHS.getValueType(), LHS, RHS, N->getFlags());
     174             : }
     175             : 
     176          14 : SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
     177          28 :   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
     178          28 :   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
     179          28 :   SDValue Op2 = GetScalarizedVector(N->getOperand(2));
     180          14 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     181          14 :                      Op0.getValueType(), Op0, Op1, Op2);
     182             : }
     183             : 
     184          36 : SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
     185          72 :   EVT VT = N->getValueType(0).getVectorElementType();
     186          36 :   unsigned NumOpers = N->getNumOperands();
     187          36 :   SDValue Chain = N->getOperand(0);
     188          36 :   EVT ValueVTs[] = {VT, MVT::Other};
     189             :   SDLoc dl(N);
     190             : 
     191             :   SmallVector<SDValue, 4> Opers;
     192             : 
     193             :   // The Chain is the first operand.
     194          36 :   Opers.push_back(Chain);
     195             : 
     196             :   // Now process the remaining operands.
     197          90 :   for (unsigned i = 1; i < NumOpers; ++i) {
     198          54 :     SDValue Oper = N->getOperand(i);
     199             : 
     200         162 :     if (Oper.getValueType().isVector())
     201          52 :       Oper = GetScalarizedVector(Oper);
     202             : 
     203          54 :     Opers.push_back(Oper);
     204             :   }
     205             : 
     206         108 :   SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
     207             : 
     208             :   // Legalize the chain result - switch anything that used the old chain to
     209             :   // use the new one.
     210          36 :   ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
     211          36 :   return Result;
     212             : }
     213             : 
     214           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
     215             :                                                        unsigned ResNo) {
     216           0 :   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
     217           0 :   return GetScalarizedVector(Op);
     218             : }
     219             : 
     220       17024 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
     221       17024 :   SDValue Op = N->getOperand(0);
     222       17024 :   if (Op.getValueType().isVector()
     223       17411 :       && Op.getValueType().getVectorNumElements() == 1
     224       17024 :       && !isSimpleLegalType(Op.getValueType()))
     225         130 :     Op = GetScalarizedVector(Op);
     226       34048 :   EVT NewVT = N->getValueType(0).getVectorElementType();
     227       17024 :   return DAG.getNode(ISD::BITCAST, SDLoc(N),
     228       17024 :                      NewVT, Op);
     229             : }
     230             : 
     231        9285 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
     232       18570 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     233        9285 :   SDValue InOp = N->getOperand(0);
     234             :   // The BUILD_VECTOR operands may be of wider element types and
     235             :   // we may need to truncate them back to the requested return type.
     236        9285 :   if (EltVT.isInteger())
     237       14498 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
     238        2036 :   return InOp;
     239             : }
     240             : 
     241        5194 : SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
     242        5194 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
     243        5194 :                      N->getValueType(0).getVectorElementType(),
     244       15582 :                      N->getOperand(0), N->getOperand(1));
     245             : }
     246             : 
     247         754 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
     248        1508 :   EVT NewVT = N->getValueType(0).getVectorElementType();
     249        1508 :   SDValue Op = GetScalarizedVector(N->getOperand(0));
     250         754 :   return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
     251         754 :                      NewVT, Op, N->getOperand(1));
     252             : }
     253             : 
     254           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
     255           0 :   SDValue Op = GetScalarizedVector(N->getOperand(0));
     256           0 :   return DAG.getNode(ISD::FPOWI, SDLoc(N),
     257           0 :                      Op.getValueType(), Op, N->getOperand(1));
     258             : }
     259             : 
     260         148 : SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
     261             :   // The value to insert may have a wider type than the vector element type,
     262             :   // so be sure to truncate it to the element type if necessary.
     263         148 :   SDValue Op = N->getOperand(1);
     264         296 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     265         172 :   if (Op.getValueType() != EltVT)
     266             :     // FIXME: Can this happen for floating point types?
     267           0 :     Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
     268         148 :   return Op;
     269             : }
     270             : 
     271        5376 : SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
     272             :   assert(N->isUnindexed() && "Indexed vector load?");
     273             : 
     274        5376 :   SDValue Result = DAG.getLoad(
     275             :       ISD::UNINDEXED, N->getExtensionType(),
     276        5376 :       N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
     277        5376 :       N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
     278        5376 :       N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
     279        5376 :       N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
     280       21504 :       N->getAAInfo());
     281             : 
     282             :   // Legalize the chain result - switch anything that used the old chain to
     283             :   // use the new one.
     284        5376 :   ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
     285        5376 :   return Result;
     286             : }
     287             : 
     288       17214 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
     289             :   // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
     290       34428 :   EVT DestVT = N->getValueType(0).getVectorElementType();
     291       17214 :   SDValue Op = N->getOperand(0);
     292       17214 :   EVT OpVT = Op.getValueType();
     293             :   SDLoc DL(N);
     294             :   // The result needs scalarizing, but it's not a given that the source does.
     295             :   // This is a workaround for targets where it's impossible to scalarize the
     296             :   // result of a conversion, because the source type is legal.
     297             :   // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
     298             :   // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
     299             :   // legal and was not scalarized.
     300             :   // See the similar logic in ScalarizeVecRes_SETCC
     301       17214 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     302       17208 :     Op = GetScalarizedVector(Op);
     303             :   } else {
     304           6 :     EVT VT = OpVT.getVectorElementType();
     305           6 :     Op = DAG.getNode(
     306             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
     307           6 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     308             :   }
     309       34428 :   return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
     310             : }
     311             : 
     312          21 : SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
     313          42 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     314          42 :   EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
     315          42 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     316          21 :   return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
     317          63 :                      LHS, DAG.getValueType(ExtVT));
     318             : }
     319             : 
     320          20 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
     321             :   SDLoc DL(N);
     322          20 :   SDValue Op = N->getOperand(0);
     323             : 
     324          20 :   EVT OpVT = Op.getValueType();
     325          20 :   EVT OpEltVT = OpVT.getVectorElementType();
     326          40 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     327             : 
     328          20 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     329           0 :     Op = GetScalarizedVector(Op);
     330             :   } else {
     331          20 :     Op = DAG.getNode(
     332             :         ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
     333          20 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     334             :   }
     335             : 
     336          40 :   switch (N->getOpcode()) {
     337          20 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     338          40 :     return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
     339           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     340           0 :     return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
     341           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     342           0 :     return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
     343             :   }
     344             : 
     345           0 :   llvm_unreachable("Illegal extend_vector_inreg opcode");
     346             : }
     347             : 
     348         151 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
     349             :   // If the operand is wider than the vector element type then it is implicitly
     350             :   // truncated.  Make that explicit here.
     351         302 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     352         151 :   SDValue InOp = N->getOperand(0);
     353         159 :   if (InOp.getValueType() != EltVT)
     354           0 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
     355         151 :   return InOp;
     356             : }
     357             : 
     358         357 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
     359         357 :   SDValue Cond = N->getOperand(0);
     360         357 :   EVT OpVT = Cond.getValueType();
     361             :   SDLoc DL(N);
     362             :   // The vselect result and true/value operands needs scalarizing, but it's
     363             :   // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
     364             :   // See the similar logic in ScalarizeVecRes_SETCC
     365         357 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     366         339 :     Cond = GetScalarizedVector(Cond);
     367             :   } else {
     368          18 :     EVT VT = OpVT.getVectorElementType();
     369          18 :     Cond = DAG.getNode(
     370             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
     371          18 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     372             :   }
     373             : 
     374         714 :   SDValue LHS = GetScalarizedVector(N->getOperand(1));
     375             :   TargetLowering::BooleanContent ScalarBool =
     376         357 :       TLI.getBooleanContents(false, false);
     377             :   TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
     378             : 
     379             :   // If integer and float booleans have different contents then we can't
     380             :   // reliably optimize in all cases. There is a full explanation for this in
     381             :   // DAGCombiner::visitSELECT() where the same issue affects folding
     382             :   // (select C, 0, 1) to (xor C, 1).
     383         357 :   if (TLI.getBooleanContents(false, false) !=
     384             :       TLI.getBooleanContents(false, true)) {
     385             :     // At least try the common case where the boolean is generated by a
     386             :     // comparison.
     387           0 :     if (Cond->getOpcode() == ISD::SETCC) {
     388           0 :       EVT OpVT = Cond->getOperand(0).getValueType();
     389           0 :       ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
     390           0 :       VecBool = TLI.getBooleanContents(OpVT);
     391             :     } else
     392             :       ScalarBool = TargetLowering::UndefinedBooleanContent;
     393             :   }
     394             : 
     395         357 :   EVT CondVT = Cond.getValueType();
     396         357 :   if (ScalarBool != VecBool) {
     397          53 :     switch (ScalarBool) {
     398             :       case TargetLowering::UndefinedBooleanContent:
     399             :         break;
     400          53 :       case TargetLowering::ZeroOrOneBooleanContent:
     401             :         assert(VecBool == TargetLowering::UndefinedBooleanContent ||
     402             :                VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
     403             :         // Vector read from all ones, scalar expects a single 1 so mask.
     404         106 :         Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
     405         106 :                            Cond, DAG.getConstant(1, SDLoc(N), CondVT));
     406          53 :         break;
     407           0 :       case TargetLowering::ZeroOrNegativeOneBooleanContent:
     408             :         assert(VecBool == TargetLowering::UndefinedBooleanContent ||
     409             :                VecBool == TargetLowering::ZeroOrOneBooleanContent);
     410             :         // Vector reads from a one, scalar from all ones so sign extend.
     411           0 :         Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
     412           0 :                            Cond, DAG.getValueType(MVT::i1));
     413           0 :         break;
     414             :     }
     415             :   }
     416             : 
     417             :   // Truncate the condition if needed
     418         357 :   auto BoolVT = getSetCCResultType(CondVT);
     419         357 :   if (BoolVT.bitsLT(CondVT))
     420           4 :     Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
     421             : 
     422         357 :   return DAG.getSelect(SDLoc(N),
     423             :                        LHS.getValueType(), Cond, LHS,
     424         714 :                        GetScalarizedVector(N->getOperand(2)));
     425             : }
     426             : 
     427         209 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
     428         418 :   SDValue LHS = GetScalarizedVector(N->getOperand(1));
     429         418 :   return DAG.getSelect(SDLoc(N),
     430         209 :                        LHS.getValueType(), N->getOperand(0), LHS,
     431         418 :                        GetScalarizedVector(N->getOperand(2)));
     432             : }
     433             : 
     434           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
     435           0 :   SDValue LHS = GetScalarizedVector(N->getOperand(2));
     436           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
     437           0 :                      N->getOperand(0), N->getOperand(1),
     438             :                      LHS, GetScalarizedVector(N->getOperand(3)),
     439           0 :                      N->getOperand(4));
     440             : }
     441             : 
     442         133 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
     443         266 :   return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
     444             : }
     445             : 
     446           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
     447             :   // Figure out if the scalar is the LHS or RHS and return it.
     448           0 :   SDValue Arg = N->getOperand(2).getOperand(0);
     449           0 :   if (Arg.isUndef())
     450           0 :     return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
     451           0 :   unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
     452           0 :   return GetScalarizedVector(N->getOperand(Op));
     453             : }
     454             : 
     455        1140 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
     456             :   assert(N->getValueType(0).isVector() &&
     457             :          N->getOperand(0).getValueType().isVector() &&
     458             :          "Operand types must be vectors");
     459        1140 :   SDValue LHS = N->getOperand(0);
     460        1140 :   SDValue RHS = N->getOperand(1);
     461        1140 :   EVT OpVT = LHS.getValueType();
     462        2280 :   EVT NVT = N->getValueType(0).getVectorElementType();
     463             :   SDLoc DL(N);
     464             : 
     465             :   // The result needs scalarizing, but it's not a given that the source does.
     466        1140 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     467        1134 :     LHS = GetScalarizedVector(LHS);
     468        1134 :     RHS = GetScalarizedVector(RHS);
     469             :   } else {
     470           6 :     EVT VT = OpVT.getVectorElementType();
     471           6 :     LHS = DAG.getNode(
     472             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
     473           6 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     474           6 :     RHS = DAG.getNode(
     475             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
     476           6 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     477             :   }
     478             : 
     479             :   // Turn it into a scalar SETCC.
     480        1140 :   SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
     481        2280 :                             N->getOperand(2));
     482             :   // Vectors may have a different boolean contents to scalars.  Promote the
     483             :   // value appropriately.
     484             :   ISD::NodeType ExtendCode =
     485        1140 :       TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
     486        2280 :   return DAG.getNode(ExtendCode, DL, NVT, Res);
     487             : }
     488             : 
     489             : 
     490             : //===----------------------------------------------------------------------===//
     491             : //  Operand Vector Scalarization <1 x ty> -> ty.
     492             : //===----------------------------------------------------------------------===//
     493             : 
     494       18568 : bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
     495             :   LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
     496             :              dbgs() << "\n");
     497       18568 :   SDValue Res = SDValue();
     498             : 
     499             :   if (!Res.getNode()) {
     500       37136 :     switch (N->getOpcode()) {
     501           0 :     default:
     502             : #ifndef NDEBUG
     503             :       dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
     504             :       N->dump(&DAG);
     505             :       dbgs() << "\n";
     506             : #endif
     507           0 :       report_fatal_error("Do not know how to scalarize this operator's "
     508             :                          "operand!\n");
     509        1813 :     case ISD::BITCAST:
     510        1813 :       Res = ScalarizeVecOp_BITCAST(N);
     511        1813 :       break;
     512          22 :     case ISD::ANY_EXTEND:
     513             :     case ISD::ZERO_EXTEND:
     514             :     case ISD::SIGN_EXTEND:
     515             :     case ISD::TRUNCATE:
     516             :     case ISD::FP_TO_SINT:
     517             :     case ISD::FP_TO_UINT:
     518             :     case ISD::SINT_TO_FP:
     519             :     case ISD::UINT_TO_FP:
     520          22 :       Res = ScalarizeVecOp_UnaryOp(N);
     521          22 :       break;
     522        6082 :     case ISD::CONCAT_VECTORS:
     523        6082 :       Res = ScalarizeVecOp_CONCAT_VECTORS(N);
     524        6082 :       break;
     525        5668 :     case ISD::EXTRACT_VECTOR_ELT:
     526        5668 :       Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
     527        5668 :       break;
     528           1 :     case ISD::VSELECT:
     529           1 :       Res = ScalarizeVecOp_VSELECT(N);
     530           1 :       break;
     531          14 :     case ISD::SETCC:
     532          14 :       Res = ScalarizeVecOp_VSETCC(N);
     533          14 :       break;
     534             :     case ISD::STORE:
     535        4965 :       Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
     536        4965 :       break;
     537           3 :     case ISD::FP_ROUND:
     538           3 :       Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
     539           3 :       break;
     540             :     }
     541             :   }
     542             : 
     543             :   // If the result is null, the sub-method took care of registering results etc.
     544       18568 :   if (!Res.getNode()) return false;
     545             : 
     546             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
     547             :   // core about this.
     548       18568 :   if (Res.getNode() == N)
     549             :     return true;
     550             : 
     551             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
     552             :          "Invalid operand expansion");
     553             : 
     554       18568 :   ReplaceValueWith(SDValue(N, 0), Res);
     555       18568 :   return false;
     556             : }
     557             : 
     558             : /// If the value to convert is a vector that needs to be scalarized, it must be
     559             : /// <1 x ty>. Convert the element instead.
     560        1813 : SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
     561        3626 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     562        1813 :   return DAG.getNode(ISD::BITCAST, SDLoc(N),
     563        1813 :                      N->getValueType(0), Elt);
     564             : }
     565             : 
     566             : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
     567             : /// Do the operation on the element instead.
     568          22 : SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
     569             :   assert(N->getValueType(0).getVectorNumElements() == 1 &&
     570             :          "Unexpected vector type!");
     571          44 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     572          22 :   SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
     573          88 :                            N->getValueType(0).getScalarType(), Elt);
     574             :   // Revectorize the result so the types line up with what the uses of this
     575             :   // expression expect.
     576          44 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
     577             : }
     578             : 
     579             : /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
     580        6082 : SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
     581        6082 :   SmallVector<SDValue, 8> Ops(N->getNumOperands());
     582       18258 :   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
     583       24352 :     Ops[i] = GetScalarizedVector(N->getOperand(i));
     584       12164 :   return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
     585             : }
     586             : 
     587             : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
     588             : /// so just return the element, ignoring the index.
     589        5668 : SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
     590        5668 :   EVT VT = N->getValueType(0);
     591       11336 :   SDValue Res = GetScalarizedVector(N->getOperand(0));
     592           0 :   if (Res.getValueType() != VT)
     593        1544 :     Res = VT.isFloatingPoint()
     594        1544 :               ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
     595        1541 :               : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
     596        5668 :   return Res;
     597             : }
     598             : 
     599             : /// If the input condition is a vector that needs to be scalarized, it must be
     600             : /// <1 x i1>, so just convert to a normal ISD::SELECT
     601             : /// (still with vector output type since that was acceptable if we got here).
     602           1 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
     603           2 :   SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
     604           1 :   EVT VT = N->getValueType(0);
     605             : 
     606           1 :   return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
     607           1 :                      N->getOperand(2));
     608             : }
     609             : 
     610             : /// If the operand is a vector that needs to be scalarized then the
     611             : /// result must be v1i1, so just convert to a scalar SETCC and wrap
     612             : /// with a scalar_to_vector since the res type is legal if we got here
     613          14 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
     614             :   assert(N->getValueType(0).isVector() &&
     615             :          N->getOperand(0).getValueType().isVector() &&
     616             :          "Operand types must be vectors");
     617             :   assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
     618             : 
     619          14 :   EVT VT = N->getValueType(0);
     620          28 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     621          28 :   SDValue RHS = GetScalarizedVector(N->getOperand(1));
     622             : 
     623          14 :   EVT OpVT = N->getOperand(0).getValueType();
     624          14 :   EVT NVT = VT.getVectorElementType();
     625             :   SDLoc DL(N);
     626             :   // Turn it into a scalar SETCC.
     627          14 :   SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
     628          28 :       N->getOperand(2));
     629             : 
     630             :   // Vectors may have a different boolean contents to scalars.  Promote the
     631             :   // value appropriately.
     632             :   ISD::NodeType ExtendCode =
     633          14 :       TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
     634             : 
     635          28 :   Res = DAG.getNode(ExtendCode, DL, NVT, Res);
     636             : 
     637          28 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
     638             : }
     639             : 
     640             : /// If the value to store is a vector that needs to be scalarized, it must be
     641             : /// <1 x ty>. Just store the element.
     642        4965 : SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
     643             :   assert(N->isUnindexed() && "Indexed store of one-element vector?");
     644             :   assert(OpNo == 1 && "Do not know how to scalarize this operand!");
     645             :   SDLoc dl(N);
     646             : 
     647        4965 :   if (N->isTruncatingStore())
     648           0 :     return DAG.getTruncStore(
     649             :         N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
     650           0 :         N->getBasePtr(), N->getPointerInfo(),
     651           0 :         N->getMemoryVT().getVectorElementType(), N->getAlignment(),
     652           0 :         N->getMemOperand()->getFlags(), N->getAAInfo());
     653             : 
     654        4965 :   return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
     655             :                       N->getBasePtr(), N->getPointerInfo(),
     656        4965 :                       N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
     657       14895 :                       N->getAAInfo());
     658             : }
     659             : 
     660             : /// If the value to round is a vector that needs to be scalarized, it must be
     661             : /// <1 x ty>. Convert the element instead.
     662           3 : SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
     663           6 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     664           3 :   SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
     665           3 :                             N->getValueType(0).getVectorElementType(), Elt,
     666           9 :                             N->getOperand(1));
     667           6 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
     668             : }
     669             : 
     670             : //===----------------------------------------------------------------------===//
     671             : //  Result Vector Splitting
     672             : //===----------------------------------------------------------------------===//
     673             : 
     674             : /// This method is called when the specified result of the specified node is
     675             : /// found to need vector splitting. At this point, the node may also have
     676             : /// invalid operands or may have other results that need legalization, we just
     677             : /// know that (at least) one result needs vector splitting.
     678       91109 : void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
     679             :   LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
     680       91109 :   SDValue Lo, Hi;
     681             : 
     682             :   // See if the target wants to custom expand this node.
     683      182218 :   if (CustomLowerNode(N, N->getValueType(ResNo), true))
     684         357 :     return;
     685             : 
     686      181504 :   switch (N->getOpcode()) {
     687           0 :   default:
     688             : #ifndef NDEBUG
     689             :     dbgs() << "SplitVectorResult #" << ResNo << ": ";
     690             :     N->dump(&DAG);
     691             :     dbgs() << "\n";
     692             : #endif
     693           0 :     report_fatal_error("Do not know how to split the result of this "
     694             :                        "operator!\n");
     695             : 
     696           0 :   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
     697        1268 :   case ISD::VSELECT:
     698        1268 :   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
     699           0 :   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
     700        1796 :   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
     701       21119 :   case ISD::BITCAST:           SplitVecRes_BITCAST(N, Lo, Hi); break;
     702       10556 :   case ISD::BUILD_VECTOR:      SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
     703       15265 :   case ISD::CONCAT_VECTORS:    SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
     704        1690 :   case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
     705          29 :   case ISD::INSERT_SUBVECTOR:  SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
     706           0 :   case ISD::FP_ROUND_INREG:    SplitVecRes_InregOp(N, Lo, Hi); break;
     707           8 :   case ISD::FPOWI:             SplitVecRes_FPOWI(N, Lo, Hi); break;
     708          11 :   case ISD::FCOPYSIGN:         SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
     709         764 :   case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
     710         210 :   case ISD::SCALAR_TO_VECTOR:  SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
     711          79 :   case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
     712             :   case ISD::LOAD:
     713        4219 :     SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
     714        4219 :     break;
     715             :   case ISD::MLOAD:
     716          20 :     SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
     717          20 :     break;
     718             :   case ISD::MGATHER:
     719          16 :     SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
     720          16 :     break;
     721        2608 :   case ISD::SETCC:
     722        2608 :     SplitVecRes_SETCC(N, Lo, Hi);
     723        2608 :     break;
     724             :   case ISD::VECTOR_SHUFFLE:
     725        5080 :     SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
     726        5080 :     break;
     727             : 
     728         403 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     729             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     730             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     731         403 :     SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
     732         403 :     break;
     733             : 
     734        3108 :   case ISD::BITREVERSE:
     735             :   case ISD::BSWAP:
     736             :   case ISD::CTLZ:
     737             :   case ISD::CTTZ:
     738             :   case ISD::CTLZ_ZERO_UNDEF:
     739             :   case ISD::CTTZ_ZERO_UNDEF:
     740             :   case ISD::CTPOP:
     741             :   case ISD::FABS:
     742             :   case ISD::FCEIL:
     743             :   case ISD::FCOS:
     744             :   case ISD::FEXP:
     745             :   case ISD::FEXP2:
     746             :   case ISD::FFLOOR:
     747             :   case ISD::FLOG:
     748             :   case ISD::FLOG10:
     749             :   case ISD::FLOG2:
     750             :   case ISD::FNEARBYINT:
     751             :   case ISD::FNEG:
     752             :   case ISD::FP_EXTEND:
     753             :   case ISD::FP_ROUND:
     754             :   case ISD::FP_TO_SINT:
     755             :   case ISD::FP_TO_UINT:
     756             :   case ISD::FRINT:
     757             :   case ISD::FROUND:
     758             :   case ISD::FSIN:
     759             :   case ISD::FSQRT:
     760             :   case ISD::FTRUNC:
     761             :   case ISD::SINT_TO_FP:
     762             :   case ISD::TRUNCATE:
     763             :   case ISD::UINT_TO_FP:
     764             :   case ISD::FCANONICALIZE:
     765        3108 :     SplitVecRes_UnaryOp(N, Lo, Hi);
     766        3108 :     break;
     767             : 
     768        7685 :   case ISD::ANY_EXTEND:
     769             :   case ISD::SIGN_EXTEND:
     770             :   case ISD::ZERO_EXTEND:
     771        7685 :     SplitVecRes_ExtendOp(N, Lo, Hi);
     772        7685 :     break;
     773             : 
     774       14634 :   case ISD::ADD:
     775             :   case ISD::SUB:
     776             :   case ISD::MUL:
     777             :   case ISD::MULHS:
     778             :   case ISD::MULHU:
     779             :   case ISD::FADD:
     780             :   case ISD::FSUB:
     781             :   case ISD::FMUL:
     782             :   case ISD::FMINNUM:
     783             :   case ISD::FMAXNUM:
     784             :   case ISD::FMINNAN:
     785             :   case ISD::FMAXNAN:
     786             :   case ISD::SDIV:
     787             :   case ISD::UDIV:
     788             :   case ISD::FDIV:
     789             :   case ISD::FPOW:
     790             :   case ISD::AND:
     791             :   case ISD::OR:
     792             :   case ISD::XOR:
     793             :   case ISD::SHL:
     794             :   case ISD::SRA:
     795             :   case ISD::SRL:
     796             :   case ISD::UREM:
     797             :   case ISD::SREM:
     798             :   case ISD::FREM:
     799             :   case ISD::SMIN:
     800             :   case ISD::SMAX:
     801             :   case ISD::UMIN:
     802             :   case ISD::UMAX:
     803       14634 :     SplitVecRes_BinOp(N, Lo, Hi);
     804       14634 :     break;
     805         165 :   case ISD::FMA:
     806         165 :     SplitVecRes_TernaryOp(N, Lo, Hi);
     807         165 :     break;
     808          19 :   case ISD::STRICT_FADD:
     809             :   case ISD::STRICT_FSUB:
     810             :   case ISD::STRICT_FMUL:
     811             :   case ISD::STRICT_FDIV:
     812             :   case ISD::STRICT_FREM:
     813             :   case ISD::STRICT_FSQRT:
     814             :   case ISD::STRICT_FMA:
     815             :   case ISD::STRICT_FPOW:
     816             :   case ISD::STRICT_FPOWI:
     817             :   case ISD::STRICT_FSIN:
     818             :   case ISD::STRICT_FCOS:
     819             :   case ISD::STRICT_FEXP:
     820             :   case ISD::STRICT_FEXP2:
     821             :   case ISD::STRICT_FLOG:
     822             :   case ISD::STRICT_FLOG10:
     823             :   case ISD::STRICT_FLOG2:
     824             :   case ISD::STRICT_FRINT:
     825             :   case ISD::STRICT_FNEARBYINT:
     826          19 :     SplitVecRes_StrictFPOp(N, Lo, Hi);
     827          19 :     break;
     828             :   }
     829             : 
     830             :   // If Lo/Hi is null, the sub-method took care of registering results etc.
     831       90752 :   if (Lo.getNode())
     832       90752 :     SetSplitVector(SDValue(N, ResNo), Lo, Hi);
     833             : }
     834             : 
     835       14634 : void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
     836             :                                          SDValue &Hi) {
     837       14634 :   SDValue LHSLo, LHSHi;
     838       29268 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
     839       14634 :   SDValue RHSLo, RHSHi;
     840       29268 :   GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
     841             :   SDLoc dl(N);
     842             : 
     843       14634 :   const SDNodeFlags Flags = N->getFlags();
     844       14634 :   unsigned Opcode = N->getOpcode();
     845       29268 :   Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
     846       29268 :   Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
     847       14634 : }
     848             : 
     849         165 : void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
     850             :                                              SDValue &Hi) {
     851         165 :   SDValue Op0Lo, Op0Hi;
     852         330 :   GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
     853         165 :   SDValue Op1Lo, Op1Hi;
     854         330 :   GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
     855         165 :   SDValue Op2Lo, Op2Hi;
     856         330 :   GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
     857             :   SDLoc dl(N);
     858             : 
     859         165 :   Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
     860         495 :                    Op0Lo, Op1Lo, Op2Lo);
     861         165 :   Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
     862         495 :                    Op0Hi, Op1Hi, Op2Hi);
     863         165 : }
     864             : 
     865       21119 : void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
     866             :                                            SDValue &Hi) {
     867             :   // We know the result is a vector.  The input may be either a vector or a
     868             :   // scalar value.
     869       21119 :   EVT LoVT, HiVT;
     870       42238 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     871             :   SDLoc dl(N);
     872             : 
     873       21119 :   SDValue InOp = N->getOperand(0);
     874             :   EVT InVT = InOp.getValueType();
     875             : 
     876             :   // Handle some special cases efficiently.
     877       21119 :   switch (getTypeAction(InVT)) {
     878             :   case TargetLowering::TypeLegal:
     879             :   case TargetLowering::TypePromoteInteger:
     880             :   case TargetLowering::TypePromoteFloat:
     881             :   case TargetLowering::TypeSoftenFloat:
     882             :   case TargetLowering::TypeScalarizeVector:
     883             :   case TargetLowering::TypeWidenVector:
     884             :     break;
     885        2103 :   case TargetLowering::TypeExpandInteger:
     886             :   case TargetLowering::TypeExpandFloat:
     887             :     // A scalar to vector conversion, where the scalar needs expansion.
     888             :     // If the vector is being split in two then we can just convert the
     889             :     // expanded pieces.
     890        2103 :     if (LoVT == HiVT) {
     891        2103 :       GetExpandedOp(InOp, Lo, Hi);
     892        2103 :       if (DAG.getDataLayout().isBigEndian())
     893             :         std::swap(Lo, Hi);
     894        4206 :       Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     895        4206 :       Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     896        2103 :       return;
     897             :     }
     898             :     break;
     899        2627 :   case TargetLowering::TypeSplitVector:
     900             :     // If the input is a vector that needs to be split, convert each split
     901             :     // piece of the input now.
     902        2627 :     GetSplitVector(InOp, Lo, Hi);
     903        5254 :     Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     904        5254 :     Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     905        2627 :     return;
     906             :   }
     907             : 
     908             :   // In the general case, convert the input to an integer and split it by hand.
     909       16389 :   EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
     910       16389 :   EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
     911       16389 :   if (DAG.getDataLayout().isBigEndian())
     912             :     std::swap(LoIntVT, HiIntVT);
     913             : 
     914       16389 :   SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
     915             : 
     916       16389 :   if (DAG.getDataLayout().isBigEndian())
     917             :     std::swap(Lo, Hi);
     918       32778 :   Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     919       32778 :   Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     920             : }
     921             : 
     922       10556 : void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
     923             :                                                 SDValue &Hi) {
     924       10556 :   EVT LoVT, HiVT;
     925             :   SDLoc dl(N);
     926       21112 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     927             :   unsigned LoNumElts = LoVT.getVectorNumElements();
     928       10556 :   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
     929       21112 :   Lo = DAG.getBuildVector(LoVT, dl, LoOps);
     930             : 
     931       21112 :   SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
     932       21112 :   Hi = DAG.getBuildVector(HiVT, dl, HiOps);
     933       10556 : }
     934             : 
     935       15265 : void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
     936             :                                                   SDValue &Hi) {
     937             :   assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
     938             :   SDLoc dl(N);
     939       15265 :   unsigned NumSubvectors = N->getNumOperands() / 2;
     940       15265 :   if (NumSubvectors == 1) {
     941       10889 :     Lo = N->getOperand(0);
     942       10889 :     Hi = N->getOperand(1);
     943             :     return;
     944             :   }
     945             : 
     946             :   EVT LoVT, HiVT;
     947        8752 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     948             : 
     949        4376 :   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
     950        8752 :   Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
     951             : 
     952        8752 :   SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
     953        8752 :   Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
     954             : }
     955             : 
     956        1690 : void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
     957             :                                                      SDValue &Hi) {
     958        1690 :   SDValue Vec = N->getOperand(0);
     959        1690 :   SDValue Idx = N->getOperand(1);
     960             :   SDLoc dl(N);
     961             : 
     962        1690 :   EVT LoVT, HiVT;
     963        5070 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     964             : 
     965        3380 :   Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
     966        1690 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
     967        1690 :   Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
     968        1690 :                    DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
     969        3380 :                                    TLI.getVectorIdxTy(DAG.getDataLayout())));
     970        1690 : }
     971             : 
     972          29 : void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
     973             :                                                     SDValue &Hi) {
     974          29 :   SDValue Vec = N->getOperand(0);
     975          29 :   SDValue SubVec = N->getOperand(1);
     976          29 :   SDValue Idx = N->getOperand(2);
     977             :   SDLoc dl(N);
     978          29 :   GetSplitVector(Vec, Lo, Hi);
     979             : 
     980          29 :   EVT VecVT = Vec.getValueType();
     981             :   unsigned VecElems = VecVT.getVectorNumElements();
     982          58 :   unsigned SubElems = SubVec.getValueType().getVectorNumElements();
     983             : 
     984             :   // If we know the index is 0, and we know the subvector doesn't cross the
     985             :   // boundary between the halves, we can avoid spilling the vector, and insert
     986             :   // into the lower half of the split vector directly.
     987             :   // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
     988             :   // the index is constant and there is no boundary crossing. But those cases
     989             :   // don't seem to get hit in practice.
     990             :   if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
     991          29 :     unsigned IdxVal = ConstIdx->getZExtValue();
     992          29 :     if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
     993             :       EVT LoVT, HiVT;
     994          58 :       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     995          58 :       Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
     996             :       return;
     997             :     }
     998             :   }
     999             : 
    1000             :   // Spill the vector to the stack.
    1001           0 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1002             :   SDValue Store =
    1003           0 :       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
    1004             : 
    1005             :   // Store the new subvector into the specified index.
    1006           0 :   SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1007           0 :   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
    1008           0 :   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
    1009           0 :   Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
    1010             : 
    1011             :   // Load the Lo part from the stack slot.
    1012           0 :   Lo =
    1013           0 :       DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
    1014             : 
    1015             :   // Increment the pointer to the other part.
    1016           0 :   unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
    1017           0 :   StackPtr =
    1018           0 :       DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
    1019           0 :                   DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
    1020             : 
    1021             :   // Load the Hi part from the stack slot.
    1022           0 :   Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
    1023           0 :                    MinAlign(Alignment, IncrementSize));
    1024             : }
    1025             : 
    1026           8 : void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
    1027             :                                          SDValue &Hi) {
    1028             :   SDLoc dl(N);
    1029          16 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1030          32 :   Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
    1031          32 :   Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
    1032           8 : }
    1033             : 
    1034          11 : void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
    1035             :                                              SDValue &Hi) {
    1036          11 :   SDValue LHSLo, LHSHi;
    1037          22 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
    1038             :   SDLoc DL(N);
    1039             : 
    1040          11 :   SDValue RHSLo, RHSHi;
    1041          11 :   SDValue RHS = N->getOperand(1);
    1042          11 :   EVT RHSVT = RHS.getValueType();
    1043          11 :   if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
    1044           9 :     GetSplitVector(RHS, RHSLo, RHSHi);
    1045             :   else
    1046           2 :     std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
    1047             : 
    1048             : 
    1049          33 :   Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
    1050          33 :   Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
    1051          11 : }
    1052             : 
    1053          79 : void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
    1054             :                                            SDValue &Hi) {
    1055          79 :   SDValue LHSLo, LHSHi;
    1056         158 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
    1057             :   SDLoc dl(N);
    1058             : 
    1059             :   EVT LoVT, HiVT;
    1060             :   std::tie(LoVT, HiVT) =
    1061         158 :     DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
    1062             : 
    1063          79 :   Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
    1064          79 :                    DAG.getValueType(LoVT));
    1065          79 :   Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
    1066          79 :                    DAG.getValueType(HiVT));
    1067          79 : }
    1068             : 
    1069         409 : void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
    1070             :                                                  SDValue &Hi) {
    1071         409 :   unsigned Opcode = N->getOpcode();
    1072         409 :   SDValue N0 = N->getOperand(0);
    1073             : 
    1074             :   SDLoc dl(N);
    1075         409 :   SDValue InLo, InHi;
    1076             : 
    1077         409 :   if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
    1078         407 :     GetSplitVector(N0, InLo, InHi);
    1079             :   else
    1080           2 :     std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
    1081             : 
    1082         818 :   EVT InLoVT = InLo.getValueType();
    1083             :   unsigned InNumElements = InLoVT.getVectorNumElements();
    1084             : 
    1085         409 :   EVT OutLoVT, OutHiVT;
    1086         818 :   std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1087             :   unsigned OutNumElements = OutLoVT.getVectorNumElements();
    1088             :   assert((2 * OutNumElements) <= InNumElements &&
    1089             :          "Illegal extend vector in reg split");
    1090             : 
    1091             :   // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
    1092             :   // input vector (i.e. we only use InLo):
    1093             :   // OutLo will extend the first OutNumElements from InLo.
    1094             :   // OutHi will extend the next OutNumElements from InLo.
    1095             : 
    1096             :   // Shuffle the elements from InLo for OutHi into the bottom elements to
    1097             :   // create a 'fake' InHi.
    1098         409 :   SmallVector<int, 8> SplitHi(InNumElements, -1);
    1099        2555 :   for (unsigned i = 0; i != OutNumElements; ++i)
    1100        4292 :     SplitHi[i] = i + OutNumElements;
    1101         818 :   InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
    1102             : 
    1103         818 :   Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
    1104         818 :   Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
    1105         409 : }
    1106             : 
    1107          19 : void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
    1108             :                                               SDValue &Hi) {
    1109          19 :   unsigned NumOps = N->getNumOperands();
    1110          19 :   SDValue Chain = N->getOperand(0);
    1111          19 :   EVT LoVT, HiVT;
    1112             :   SDLoc dl(N);
    1113          38 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1114             : 
    1115             :   SmallVector<SDValue, 4> OpsLo;
    1116             :   SmallVector<SDValue, 4> OpsHi;
    1117             : 
    1118             :   // The Chain is the first operand.
    1119          19 :   OpsLo.push_back(Chain);
    1120          19 :   OpsHi.push_back(Chain);
    1121             : 
    1122             :   // Now process the remaining operands.
    1123          49 :   for (unsigned i = 1; i < NumOps; ++i) {
    1124          30 :     SDValue Op = N->getOperand(i);
    1125          30 :     SDValue OpLo = Op;
    1126          30 :     SDValue OpHi = Op;
    1127             : 
    1128          30 :     EVT InVT = Op.getValueType();
    1129          30 :     if (InVT.isVector()) {
    1130             :       // If the input also splits, handle it directly for a
    1131             :       // compile time speedup. Otherwise split it by hand.
    1132          29 :       if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
    1133          29 :         GetSplitVector(Op, OpLo, OpHi);
    1134             :       else
    1135           0 :         std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
    1136             :     }
    1137             : 
    1138          30 :     OpsLo.push_back(OpLo);
    1139          30 :     OpsHi.push_back(OpHi);
    1140             :   }
    1141             : 
    1142          19 :   EVT LoValueVTs[] = {LoVT, MVT::Other};
    1143          19 :   EVT HiValueVTs[] = {HiVT, MVT::Other};
    1144          57 :   Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
    1145          57 :   Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
    1146             : 
    1147             :   // Build a factor node to remember that this Op is independent of the
    1148             :   // other one.
    1149          19 :   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
    1150          38 :                       Lo.getValue(1), Hi.getValue(1));
    1151             : 
    1152             :   // Legalize the chain result - switch anything that used the old chain to
    1153             :   // use the new one.
    1154          19 :   ReplaceValueWith(SDValue(N, 1), Chain);
    1155          19 : }
    1156             : 
    1157         764 : void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
    1158             :                                                      SDValue &Hi) {
    1159         764 :   SDValue Vec = N->getOperand(0);
    1160         764 :   SDValue Elt = N->getOperand(1);
    1161         764 :   SDValue Idx = N->getOperand(2);
    1162             :   SDLoc dl(N);
    1163         764 :   GetSplitVector(Vec, Lo, Hi);
    1164             : 
    1165             :   if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
    1166         715 :     unsigned IdxVal = CIdx->getZExtValue();
    1167        1430 :     unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
    1168         715 :     if (IdxVal < LoNumElts)
    1169         405 :       Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
    1170         405 :                        Lo.getValueType(), Lo, Elt, Idx);
    1171             :     else
    1172         310 :       Hi =
    1173         310 :           DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
    1174         310 :                       DAG.getConstant(IdxVal - LoNumElts, dl,
    1175         310 :                                       TLI.getVectorIdxTy(DAG.getDataLayout())));
    1176         715 :     return;
    1177             :   }
    1178             : 
    1179             :   // See if the target wants to custom expand this node.
    1180          98 :   if (CustomLowerNode(N, N->getValueType(0), true))
    1181             :     return;
    1182             : 
    1183             :   // Make the vector elements byte-addressable if they aren't already.
    1184          49 :   EVT VecVT = Vec.getValueType();
    1185          49 :   EVT EltVT = VecVT.getVectorElementType();
    1186          49 :   if (VecVT.getScalarSizeInBits() < 8) {
    1187           8 :     EltVT = MVT::i8;
    1188           8 :     VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
    1189           8 :                              VecVT.getVectorNumElements());
    1190          16 :     Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
    1191             :     // Extend the element type to match if needed.
    1192           8 :     if (EltVT.bitsGT(Elt.getValueType()))
    1193          16 :       Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
    1194             :   }
    1195             : 
    1196             :   // Spill the vector to the stack.
    1197          49 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1198          49 :   auto &MF = DAG.getMachineFunction();
    1199          49 :   auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1200          49 :   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
    1201          98 :   SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
    1202             : 
    1203             :   // Store the new element.  This may be larger than the vector element type,
    1204             :   // so use a truncating store.
    1205          49 :   SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1206          49 :   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
    1207          49 :   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
    1208          49 :   Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
    1209          49 :                             MachinePointerInfo::getUnknownStack(MF), EltVT);
    1210             : 
    1211          49 :   EVT LoVT, HiVT;
    1212          49 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
    1213             : 
    1214             :   // Load the Lo part from the stack slot.
    1215          98 :   Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
    1216             : 
    1217             :   // Increment the pointer to the other part.
    1218          49 :   unsigned IncrementSize = LoVT.getSizeInBits() / 8;
    1219          49 :   StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
    1220             :                          DAG.getConstant(IncrementSize, dl,
    1221          49 :                                          StackPtr.getValueType()));
    1222             : 
    1223             :   // Load the Hi part from the stack slot.
    1224          49 :   Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
    1225             :                    PtrInfo.getWithOffset(IncrementSize),
    1226          49 :                    MinAlign(Alignment, IncrementSize));
    1227             : 
    1228             :   // If we adjusted the original type, we need to truncate the results.
    1229          98 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1230         106 :   if (LoVT != Lo.getValueType())
    1231          16 :     Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
    1232         106 :   if (HiVT != Hi.getValueType())
    1233          16 :     Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
    1234             : }
    1235             : 
    1236         210 : void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
    1237             :                                                     SDValue &Hi) {
    1238         210 :   EVT LoVT, HiVT;
    1239             :   SDLoc dl(N);
    1240         420 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1241         630 :   Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
    1242         210 :   Hi = DAG.getUNDEF(HiVT);
    1243         210 : }
    1244             : 
    1245        4219 : void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
    1246             :                                         SDValue &Hi) {
    1247             :   assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
    1248        4219 :   EVT LoVT, HiVT;
    1249             :   SDLoc dl(LD);
    1250        8438 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
    1251             : 
    1252             :   ISD::LoadExtType ExtType = LD->getExtensionType();
    1253        4219 :   SDValue Ch = LD->getChain();
    1254        4219 :   SDValue Ptr = LD->getBasePtr();
    1255        8438 :   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
    1256        4219 :   EVT MemoryVT = LD->getMemoryVT();
    1257        4219 :   unsigned Alignment = LD->getOriginalAlignment();
    1258        4219 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    1259             :   AAMDNodes AAInfo = LD->getAAInfo();
    1260             : 
    1261        4219 :   EVT LoMemVT, HiMemVT;
    1262        4219 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1263             : 
    1264        8438 :   Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
    1265        8438 :                    LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
    1266             : 
    1267        4219 :   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
    1268        4219 :   Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
    1269        8438 :   Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
    1270        4219 :                    LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
    1271        4219 :                    Alignment, MMOFlags, AAInfo);
    1272             : 
    1273             :   // Build a factor node to remember that this load is independent of the
    1274             :   // other one.
    1275        4219 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1276        8438 :                    Hi.getValue(1));
    1277             : 
    1278             :   // Legalize the chain result - switch anything that used the old chain to
    1279             :   // use the new one.
    1280        4219 :   ReplaceValueWith(SDValue(LD, 1), Ch);
    1281        4219 : }
    1282             : 
    1283          20 : void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
    1284             :                                          SDValue &Lo, SDValue &Hi) {
    1285          20 :   EVT LoVT, HiVT;
    1286             :   SDLoc dl(MLD);
    1287          40 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
    1288             : 
    1289          20 :   SDValue Ch = MLD->getChain();
    1290          20 :   SDValue Ptr = MLD->getBasePtr();
    1291          20 :   SDValue Mask = MLD->getMask();
    1292          20 :   SDValue PassThru = MLD->getPassThru();
    1293          20 :   unsigned Alignment = MLD->getOriginalAlignment();
    1294             :   ISD::LoadExtType ExtType = MLD->getExtensionType();
    1295             : 
    1296             :   // if Alignment is equal to the vector size,
    1297             :   // take the half of it for the second part
    1298             :   unsigned SecondHalfAlignment =
    1299          40 :     (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
    1300             :      Alignment/2 : Alignment;
    1301             : 
    1302             :   // Split Mask operand
    1303          20 :   SDValue MaskLo, MaskHi;
    1304          40 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1305           1 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1306             :   else
    1307          19 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1308             : 
    1309          20 :   EVT MemoryVT = MLD->getMemoryVT();
    1310          20 :   EVT LoMemVT, HiMemVT;
    1311          20 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1312             : 
    1313          20 :   SDValue PassThruLo, PassThruHi;
    1314          40 :   if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
    1315          20 :     GetSplitVector(PassThru, PassThruLo, PassThruHi);
    1316             :   else
    1317           0 :     std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
    1318             : 
    1319          20 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1320          60 :     getMachineMemOperand(MLD->getPointerInfo(),
    1321             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1322          20 :                          Alignment, MLD->getAAInfo(), MLD->getRanges());
    1323             : 
    1324          20 :   Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
    1325          20 :                          ExtType, MLD->isExpandingLoad());
    1326             : 
    1327          20 :   Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
    1328          20 :                                    MLD->isExpandingLoad());
    1329             :   unsigned HiOffset = LoMemVT.getStoreSize();
    1330             : 
    1331          60 :   MMO = DAG.getMachineFunction().getMachineMemOperand(
    1332             :       MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
    1333          20 :       HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
    1334             :       MLD->getRanges());
    1335             : 
    1336          20 :   Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
    1337          20 :                          ExtType, MLD->isExpandingLoad());
    1338             : 
    1339             :   // Build a factor node to remember that this load is independent of the
    1340             :   // other one.
    1341          20 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1342          40 :                    Hi.getValue(1));
    1343             : 
    1344             :   // Legalize the chain result - switch anything that used the old chain to
    1345             :   // use the new one.
    1346          20 :   ReplaceValueWith(SDValue(MLD, 1), Ch);
    1347             : 
    1348          20 : }
    1349             : 
    1350          16 : void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
    1351             :                                          SDValue &Lo, SDValue &Hi) {
    1352          16 :   EVT LoVT, HiVT;
    1353             :   SDLoc dl(MGT);
    1354          32 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
    1355             : 
    1356          16 :   SDValue Ch = MGT->getChain();
    1357          16 :   SDValue Ptr = MGT->getBasePtr();
    1358          16 :   SDValue Mask = MGT->getMask();
    1359          16 :   SDValue PassThru = MGT->getPassThru();
    1360          16 :   SDValue Index = MGT->getIndex();
    1361          16 :   SDValue Scale = MGT->getScale();
    1362          16 :   unsigned Alignment = MGT->getOriginalAlignment();
    1363             : 
    1364             :   // Split Mask operand
    1365          16 :   SDValue MaskLo, MaskHi;
    1366          32 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1367           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1368             :   else
    1369          16 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1370             : 
    1371          16 :   EVT MemoryVT = MGT->getMemoryVT();
    1372          16 :   EVT LoMemVT, HiMemVT;
    1373             :   // Split MemoryVT
    1374          16 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1375             : 
    1376          16 :   SDValue PassThruLo, PassThruHi;
    1377          32 :   if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
    1378          16 :     GetSplitVector(PassThru, PassThruLo, PassThruHi);
    1379             :   else
    1380           0 :     std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
    1381             : 
    1382          16 :   SDValue IndexHi, IndexLo;
    1383          32 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1384          12 :     GetSplitVector(Index, IndexLo, IndexHi);
    1385             :   else
    1386           4 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
    1387             : 
    1388          16 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1389          48 :     getMachineMemOperand(MGT->getPointerInfo(),
    1390             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1391          16 :                          Alignment, MGT->getAAInfo(), MGT->getRanges());
    1392             : 
    1393          16 :   SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
    1394          16 :   Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
    1395          16 :                            MMO);
    1396             : 
    1397          16 :   SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
    1398          16 :   Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
    1399          16 :                            MMO);
    1400             : 
    1401             :   // Build a factor node to remember that this load is independent of the
    1402             :   // other one.
    1403          16 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1404          32 :                    Hi.getValue(1));
    1405             : 
    1406             :   // Legalize the chain result - switch anything that used the old chain to
    1407             :   // use the new one.
    1408          16 :   ReplaceValueWith(SDValue(MGT, 1), Ch);
    1409          16 : }
    1410             : 
    1411             : 
    1412        2608 : void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
    1413             :   assert(N->getValueType(0).isVector() &&
    1414             :          N->getOperand(0).getValueType().isVector() &&
    1415             :          "Operand types must be vectors");
    1416             : 
    1417        2608 :   EVT LoVT, HiVT;
    1418             :   SDLoc DL(N);
    1419        5216 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1420             : 
    1421             :   // If the input also splits, handle it directly. Otherwise split it by hand.
    1422        2608 :   SDValue LL, LH, RL, RH;
    1423        5216 :   if (getTypeAction(N->getOperand(0).getValueType()) ==
    1424             :       TargetLowering::TypeSplitVector)
    1425        4822 :     GetSplitVector(N->getOperand(0), LL, LH);
    1426             :   else
    1427         197 :     std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
    1428             : 
    1429        5216 :   if (getTypeAction(N->getOperand(1).getValueType()) ==
    1430             :       TargetLowering::TypeSplitVector)
    1431        4822 :     GetSplitVector(N->getOperand(1), RL, RH);
    1432             :   else
    1433         197 :     std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
    1434             : 
    1435       10432 :   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
    1436       10432 :   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
    1437        2608 : }
    1438             : 
    1439       10725 : void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
    1440             :                                            SDValue &Hi) {
    1441             :   // Get the dest types - they may not match the input types, e.g. int_to_fp.
    1442       10725 :   EVT LoVT, HiVT;
    1443             :   SDLoc dl(N);
    1444       21450 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1445             : 
    1446             :   // If the input also splits, handle it directly for a compile time speedup.
    1447             :   // Otherwise split it by hand.
    1448       10725 :   EVT InVT = N->getOperand(0).getValueType();
    1449       10725 :   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
    1450       12944 :     GetSplitVector(N->getOperand(0), Lo, Hi);
    1451             :   else
    1452        4253 :     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
    1453             : 
    1454       21450 :   if (N->getOpcode() == ISD::FP_ROUND) {
    1455        1809 :     Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
    1456        2412 :     Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
    1457             :   } else {
    1458       20244 :     Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
    1459       30366 :     Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
    1460             :   }
    1461       10725 : }
    1462             : 
    1463        7685 : void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
    1464             :                                             SDValue &Hi) {
    1465             :   SDLoc dl(N);
    1466        7685 :   EVT SrcVT = N->getOperand(0).getValueType();
    1467        7685 :   EVT DestVT = N->getValueType(0);
    1468             :   EVT LoVT, HiVT;
    1469        7685 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
    1470             : 
    1471             :   // We can do better than a generic split operation if the extend is doing
    1472             :   // more than just doubling the width of the elements and the following are
    1473             :   // true:
    1474             :   //   - The number of vector elements is even,
    1475             :   //   - the source type is legal,
    1476             :   //   - the type of a split source is illegal,
    1477             :   //   - the type of an extended (by doubling element size) source is legal, and
    1478             :   //   - the type of that extended source when split is legal.
    1479             :   //
    1480             :   // This won't necessarily completely legalize the operation, but it will
    1481             :   // more effectively move in the right direction and prevent falling down
    1482             :   // to scalarization in many cases due to the input vector being split too
    1483             :   // far.
    1484             :   unsigned NumElements = SrcVT.getVectorNumElements();
    1485        7685 :   if ((NumElements & 1) == 0 &&
    1486        7685 :       SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
    1487        5045 :     LLVMContext &Ctx = *DAG.getContext();
    1488        5045 :     EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
    1489        5045 :     EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
    1490             : 
    1491             :     EVT SplitLoVT, SplitHiVT;
    1492        5045 :     std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
    1493        5396 :     if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
    1494         159 :         TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
    1495             :       LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
    1496             :                  N->dump(&DAG); dbgs() << "\n");
    1497             :       // Extend the source vector by one step.
    1498             :       SDValue NewSrc =
    1499         272 :           DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
    1500             :       // Get the low and high halves of the new, extended one step, vector.
    1501          68 :       std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
    1502             :       // Extend those vector halves the rest of the way.
    1503         204 :       Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
    1504         204 :       Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
    1505             :       return;
    1506             :     }
    1507             :   }
    1508             :   // Fall back to the generic unary operator splitting otherwise.
    1509        7617 :   SplitVecRes_UnaryOp(N, Lo, Hi);
    1510             : }
    1511             : 
    1512        5080 : void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
    1513             :                                                   SDValue &Lo, SDValue &Hi) {
    1514             :   // The low and high parts of the original input give four input vectors.
    1515        5080 :   SDValue Inputs[4];
    1516             :   SDLoc dl(N);
    1517       10160 :   GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
    1518       10160 :   GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
    1519       10160 :   EVT NewVT = Inputs[0].getValueType();
    1520             :   unsigned NewElts = NewVT.getVectorNumElements();
    1521             : 
    1522             :   // If Lo or Hi uses elements from at most two of the four input vectors, then
    1523             :   // express it as a vector shuffle of those two inputs.  Otherwise extract the
    1524             :   // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
    1525             :   SmallVector<int, 16> Ops;
    1526       15240 :   for (unsigned High = 0; High < 2; ++High) {
    1527       10160 :     SDValue &Output = High ? Hi : Lo;
    1528             : 
    1529             :     // Build a shuffle mask for the output, discovering on the fly which
    1530             :     // input vectors to use as shuffle operands (recorded in InputUsed).
    1531             :     // If building a suitable shuffle vector proves too hard, then bail
    1532             :     // out with useBuildVector set.
    1533       10160 :     unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
    1534       10160 :     unsigned FirstMaskIdx = High * NewElts;
    1535             :     bool useBuildVector = false;
    1536      151709 :     for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
    1537             :       // The mask element.  This indexes into the input.
    1538      141628 :       int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
    1539             : 
    1540             :       // The input vector this mask element indexes into.
    1541      141628 :       unsigned Input = (unsigned)Idx / NewElts;
    1542             : 
    1543      141628 :       if (Input >= array_lengthof(Inputs)) {
    1544             :         // The mask element does not index into any input vector.
    1545      105676 :         Ops.push_back(-1);
    1546      105676 :         continue;
    1547             :       }
    1548             : 
    1549             :       // Turn the index into an offset from the start of the input vector.
    1550       35952 :       Idx -= Input * NewElts;
    1551             : 
    1552             :       // Find or create a shuffle vector operand to hold this input.
    1553             :       unsigned OpNo;
    1554       41809 :       for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
    1555       41730 :         if (InputUsed[OpNo] == Input) {
    1556             :           // This input vector is already an operand.
    1557             :           break;
    1558       13042 :         } else if (InputUsed[OpNo] == -1U) {
    1559             :           // Create a new operand for this input vector.
    1560        7185 :           InputUsed[OpNo] = Input;
    1561        7185 :           break;
    1562             :         }
    1563             :       }
    1564             : 
    1565       35952 :       if (OpNo >= array_lengthof(InputUsed)) {
    1566             :         // More than two input vectors used!  Give up on trying to create a
    1567             :         // shuffle vector.  Insert all elements into a BUILD_VECTOR instead.
    1568             :         useBuildVector = true;
    1569             :         break;
    1570             :       }
    1571             : 
    1572             :       // Add the mask index for the new shuffle vector.
    1573       35873 :       Ops.push_back(Idx + OpNo * NewElts);
    1574             :     }
    1575             : 
    1576       10160 :     if (useBuildVector) {
    1577          79 :       EVT EltVT = NewVT.getVectorElementType();
    1578             :       SmallVector<SDValue, 16> SVOps;
    1579             : 
    1580             :       // Extract the input elements by hand.
    1581         691 :       for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
    1582             :         // The mask element.  This indexes into the input.
    1583         612 :         int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
    1584             : 
    1585             :         // The input vector this mask element indexes into.
    1586         612 :         unsigned Input = (unsigned)Idx / NewElts;
    1587             : 
    1588         612 :         if (Input >= array_lengthof(Inputs)) {
    1589             :           // The mask element is "undef" or indexes off the end of the input.
    1590         123 :           SVOps.push_back(DAG.getUNDEF(EltVT));
    1591         123 :           continue;
    1592             :         }
    1593             : 
    1594             :         // Turn the index into an offset from the start of the input vector.
    1595         489 :         Idx -= Input * NewElts;
    1596             : 
    1597             :         // Extract the vector element by hand.
    1598         978 :         SVOps.push_back(DAG.getNode(
    1599             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
    1600         978 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    1601             :       }
    1602             : 
    1603             :       // Construct the Lo/Hi output using a BUILD_VECTOR.
    1604         158 :       Output = DAG.getBuildVector(NewVT, dl, SVOps);
    1605       10081 :     } else if (InputUsed[0] == -1U) {
    1606             :       // No input vectors were used!  The result is undefined.
    1607        4124 :       Output = DAG.getUNDEF(NewVT);
    1608             :     } else {
    1609        5957 :       SDValue Op0 = Inputs[InputUsed[0]];
    1610             :       // If only one input was used, use an undefined vector for the other.
    1611        5957 :       SDValue Op1 = InputUsed[1] == -1U ?
    1612        5957 :         DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
    1613             :       // At least one input vector was used.  Create a new shuffle vector.
    1614       11914 :       Output =  DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
    1615             :     }
    1616             : 
    1617             :     Ops.clear();
    1618             :   }
    1619        5080 : }
    1620             : 
    1621             : 
    1622             : //===----------------------------------------------------------------------===//
    1623             : //  Operand Vector Splitting
    1624             : //===----------------------------------------------------------------------===//
    1625             : 
    1626             : /// This method is called when the specified operand of the specified node is
    1627             : /// found to need vector splitting. At this point, all of the result types of
    1628             : /// the node are known to be legal, but other operands of the node may need
    1629             : /// legalization as well as the specified one.
    1630       72196 : bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
    1631             :   LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
    1632       72196 :   SDValue Res = SDValue();
    1633             : 
    1634             :   // See if the target wants to custom split this node.
    1635      216588 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    1636             :     return false;
    1637             : 
    1638             :   if (!Res.getNode()) {
    1639      141662 :     switch (N->getOpcode()) {
    1640           0 :     default:
    1641             : #ifndef NDEBUG
    1642             :       dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
    1643             :       N->dump(&DAG);
    1644             :       dbgs() << "\n";
    1645             : #endif
    1646           0 :       report_fatal_error("Do not know how to split this operator's "
    1647             :                          "operand!\n");
    1648             : 
    1649         142 :     case ISD::SETCC:             Res = SplitVecOp_VSETCC(N); break;
    1650        5446 :     case ISD::BITCAST:           Res = SplitVecOp_BITCAST(N); break;
    1651       22736 :     case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
    1652       22062 :     case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
    1653           2 :     case ISD::CONCAT_VECTORS:    Res = SplitVecOp_CONCAT_VECTORS(N); break;
    1654        1898 :     case ISD::TRUNCATE:
    1655        1898 :       Res = SplitVecOp_TruncateHelper(N);
    1656        1898 :       break;
    1657          82 :     case ISD::FP_ROUND:          Res = SplitVecOp_FP_ROUND(N); break;
    1658           8 :     case ISD::FCOPYSIGN:         Res = SplitVecOp_FCOPYSIGN(N); break;
    1659             :     case ISD::STORE:
    1660        8785 :       Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
    1661        8785 :       break;
    1662             :     case ISD::MSTORE:
    1663           8 :       Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
    1664           8 :       break;
    1665             :     case ISD::MSCATTER:
    1666          22 :       Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
    1667          22 :       break;
    1668             :     case ISD::MGATHER:
    1669          33 :       Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
    1670          33 :       break;
    1671         136 :     case ISD::VSELECT:
    1672         136 :       Res = SplitVecOp_VSELECT(N, OpNo);
    1673         136 :       break;
    1674          51 :     case ISD::FP_TO_SINT:
    1675             :     case ISD::FP_TO_UINT:
    1676         153 :       if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
    1677          39 :         Res = SplitVecOp_TruncateHelper(N);
    1678             :       else
    1679          12 :         Res = SplitVecOp_UnaryOp(N);
    1680             :       break;
    1681         229 :     case ISD::SINT_TO_FP:
    1682             :     case ISD::UINT_TO_FP:
    1683         687 :       if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
    1684         123 :         Res = SplitVecOp_TruncateHelper(N);
    1685             :       else
    1686         106 :         Res = SplitVecOp_UnaryOp(N);
    1687             :       break;
    1688        9172 :     case ISD::CTTZ:
    1689             :     case ISD::CTLZ:
    1690             :     case ISD::CTPOP:
    1691             :     case ISD::FP_EXTEND:
    1692             :     case ISD::SIGN_EXTEND:
    1693             :     case ISD::ZERO_EXTEND:
    1694             :     case ISD::ANY_EXTEND:
    1695             :     case ISD::FTRUNC:
    1696             :     case ISD::FCANONICALIZE:
    1697        9172 :       Res = SplitVecOp_UnaryOp(N);
    1698        9172 :       break;
    1699             : 
    1700           6 :     case ISD::ANY_EXTEND_VECTOR_INREG:
    1701             :     case ISD::SIGN_EXTEND_VECTOR_INREG:
    1702             :     case ISD::ZERO_EXTEND_VECTOR_INREG:
    1703           6 :       Res = SplitVecOp_ExtVecInRegOp(N);
    1704           6 :       break;
    1705             : 
    1706          13 :     case ISD::VECREDUCE_FADD:
    1707             :     case ISD::VECREDUCE_FMUL:
    1708             :     case ISD::VECREDUCE_ADD:
    1709             :     case ISD::VECREDUCE_MUL:
    1710             :     case ISD::VECREDUCE_AND:
    1711             :     case ISD::VECREDUCE_OR:
    1712             :     case ISD::VECREDUCE_XOR:
    1713             :     case ISD::VECREDUCE_SMAX:
    1714             :     case ISD::VECREDUCE_SMIN:
    1715             :     case ISD::VECREDUCE_UMAX:
    1716             :     case ISD::VECREDUCE_UMIN:
    1717             :     case ISD::VECREDUCE_FMAX:
    1718             :     case ISD::VECREDUCE_FMIN:
    1719          13 :       Res = SplitVecOp_VECREDUCE(N, OpNo);
    1720          13 :       break;
    1721             :     }
    1722             :   }
    1723             : 
    1724             :   // If the result is null, the sub-method took care of registering results etc.
    1725       70831 :   if (!Res.getNode()) return false;
    1726             : 
    1727             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    1728             :   // core about this.
    1729       70798 :   if (Res.getNode() == N)
    1730             :     return true;
    1731             : 
    1732             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    1733             :          "Invalid operand expansion");
    1734             : 
    1735       49530 :   ReplaceValueWith(SDValue(N, 0), Res);
    1736       49530 :   return false;
    1737             : }
    1738             : 
    1739         140 : SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
    1740             :   // The only possibility for an illegal operand is the mask, since result type
    1741             :   // legalization would have handled this node already otherwise.
    1742             :   assert(OpNo == 0 && "Illegal operand must be mask");
    1743             : 
    1744         140 :   SDValue Mask = N->getOperand(0);
    1745         140 :   SDValue Src0 = N->getOperand(1);
    1746         140 :   SDValue Src1 = N->getOperand(2);
    1747         280 :   EVT Src0VT = Src0.getValueType();
    1748             :   SDLoc DL(N);
    1749             :   assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
    1750             : 
    1751         140 :   SDValue Lo, Hi;
    1752         280 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1753             :   assert(Lo.getValueType() == Hi.getValueType() &&
    1754             :          "Lo and Hi have differing types");
    1755             : 
    1756             :   EVT LoOpVT, HiOpVT;
    1757         140 :   std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
    1758             :   assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
    1759             : 
    1760             :   SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
    1761         140 :   std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
    1762         140 :   std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
    1763         140 :   std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
    1764             : 
    1765             :   SDValue LoSelect =
    1766         280 :     DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
    1767             :   SDValue HiSelect =
    1768         280 :     DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
    1769             : 
    1770         280 :   return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
    1771             : }
    1772             : 
    1773          13 : SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
    1774          13 :   EVT ResVT = N->getValueType(0);
    1775          13 :   SDValue Lo, Hi;
    1776             :   SDLoc dl(N);
    1777             : 
    1778          26 :   SDValue VecOp = N->getOperand(OpNo);
    1779          13 :   EVT VecVT = VecOp.getValueType();
    1780             :   assert(VecVT.isVector() && "Can only split reduce vector operand");
    1781          13 :   GetSplitVector(VecOp, Lo, Hi);
    1782             :   EVT LoOpVT, HiOpVT;
    1783          13 :   std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
    1784             : 
    1785          13 :   bool NoNaN = N->getFlags().hasNoNaNs();
    1786             :   unsigned CombineOpc = 0;
    1787          26 :   switch (N->getOpcode()) {
    1788             :   case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
    1789           0 :   case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
    1790           5 :   case ISD::VECREDUCE_ADD:  CombineOpc = ISD::ADD; break;
    1791           0 :   case ISD::VECREDUCE_MUL:  CombineOpc = ISD::MUL; break;
    1792           0 :   case ISD::VECREDUCE_AND:  CombineOpc = ISD::AND; break;
    1793           0 :   case ISD::VECREDUCE_OR:   CombineOpc = ISD::OR; break;
    1794           0 :   case ISD::VECREDUCE_XOR:  CombineOpc = ISD::XOR; break;
    1795           2 :   case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
    1796           2 :   case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
    1797           2 :   case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
    1798           2 :   case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
    1799           0 :   case ISD::VECREDUCE_FMAX:
    1800           0 :     CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
    1801             :     break;
    1802           0 :   case ISD::VECREDUCE_FMIN:
    1803           0 :     CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
    1804             :     break;
    1805           0 :   default:
    1806           0 :     llvm_unreachable("Unexpected reduce ISD node");
    1807             :   }
    1808             : 
    1809             :   // Use the appropriate scalar instruction on the split subvectors before
    1810             :   // reducing the now partially reduced smaller vector.
    1811          26 :   SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
    1812          39 :   return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
    1813             : }
    1814             : 
    1815       10676 : SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
    1816             :   // The result has a legal vector type, but the input needs splitting.
    1817       10676 :   EVT ResVT = N->getValueType(0);
    1818       10676 :   SDValue Lo, Hi;
    1819             :   SDLoc dl(N);
    1820       21352 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1821       21352 :   EVT InVT = Lo.getValueType();
    1822             : 
    1823       10676 :   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
    1824       10676 :                                InVT.getVectorNumElements());
    1825             : 
    1826       32028 :   Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
    1827       32028 :   Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
    1828             : 
    1829       21352 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
    1830             : }
    1831             : 
    1832        5446 : SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
    1833             :   // For example, i64 = BITCAST v4i16 on alpha.  Typically the vector will
    1834             :   // end up being split all the way down to individual components.  Convert the
    1835             :   // split pieces into integers and reassemble.
    1836        5446 :   SDValue Lo, Hi;
    1837       10892 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1838        5446 :   Lo = BitConvertToInteger(Lo);
    1839        5446 :   Hi = BitConvertToInteger(Hi);
    1840             : 
    1841        5446 :   if (DAG.getDataLayout().isBigEndian())
    1842             :     std::swap(Lo, Hi);
    1843             : 
    1844        5446 :   return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
    1845        5446 :                      JoinIntegers(Lo, Hi));
    1846             : }
    1847             : 
    1848       22736 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
    1849             :   // We know that the extracted result type is legal.
    1850       22736 :   EVT SubVT = N->getValueType(0);
    1851       22736 :   SDValue Idx = N->getOperand(1);
    1852             :   SDLoc dl(N);
    1853       22736 :   SDValue Lo, Hi;
    1854       45472 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1855             : 
    1856       45472 :   uint64_t LoElts = Lo.getValueType().getVectorNumElements();
    1857       22736 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    1858             : 
    1859       22736 :   if (IdxVal < LoElts) {
    1860             :     assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
    1861             :            "Extracted subvector crosses vector split!");
    1862       23552 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
    1863             :   } else {
    1864       10960 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
    1865             :                        DAG.getConstant(IdxVal - LoElts, dl,
    1866       10960 :                                        Idx.getValueType()));
    1867             :   }
    1868             : }
    1869             : 
    1870       22062 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    1871       22062 :   SDValue Vec = N->getOperand(0);
    1872       22062 :   SDValue Idx = N->getOperand(1);
    1873       22062 :   EVT VecVT = Vec.getValueType();
    1874             : 
    1875             :   if (isa<ConstantSDNode>(Idx)) {
    1876       21844 :     uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    1877             :     assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
    1878             : 
    1879       21844 :     SDValue Lo, Hi;
    1880       21844 :     GetSplitVector(Vec, Lo, Hi);
    1881             : 
    1882       43688 :     uint64_t LoElts = Lo.getValueType().getVectorNumElements();
    1883             : 
    1884       21844 :     if (IdxVal < LoElts)
    1885       12697 :       return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
    1886        9147 :     return SDValue(DAG.UpdateNodeOperands(N, Hi,
    1887        9147 :                                   DAG.getConstant(IdxVal - LoElts, SDLoc(N),
    1888        9147 :                                                   Idx.getValueType())), 0);
    1889             :   }
    1890             : 
    1891             :   // See if the target wants to custom expand this node.
    1892         436 :   if (CustomLowerNode(N, N->getValueType(0), true))
    1893           0 :     return SDValue();
    1894             : 
    1895             :   // Make the vector elements byte-addressable if they aren't already.
    1896             :   SDLoc dl(N);
    1897         218 :   EVT EltVT = VecVT.getVectorElementType();
    1898         218 :   if (VecVT.getScalarSizeInBits() < 8) {
    1899           2 :     EltVT = MVT::i8;
    1900           2 :     VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
    1901           2 :                              VecVT.getVectorNumElements());
    1902           4 :     Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
    1903             :   }
    1904             : 
    1905             :   // Store the vector to the stack.
    1906         218 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1907         218 :   auto &MF = DAG.getMachineFunction();
    1908         218 :   auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1909         218 :   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
    1910         436 :   SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
    1911             : 
    1912             :   // Load back the required element.
    1913         218 :   StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1914         218 :   return DAG.getExtLoad(
    1915             :       ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
    1916         218 :       MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
    1917             : }
    1918             : 
    1919           6 : SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
    1920           6 :   SDValue Lo, Hi;
    1921             : 
    1922             :   // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
    1923             :   // splitting the result has the same effect as splitting the input operand.
    1924           6 :   SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
    1925             : 
    1926          12 :   return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
    1927             : }
    1928             : 
    1929          33 : SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
    1930             :                                              unsigned OpNo) {
    1931          33 :   EVT LoVT, HiVT;
    1932             :   SDLoc dl(MGT);
    1933          66 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
    1934             : 
    1935          33 :   SDValue Ch = MGT->getChain();
    1936          33 :   SDValue Ptr = MGT->getBasePtr();
    1937          33 :   SDValue Index = MGT->getIndex();
    1938          33 :   SDValue Scale = MGT->getScale();
    1939          33 :   SDValue Mask = MGT->getMask();
    1940          33 :   SDValue PassThru = MGT->getPassThru();
    1941          33 :   unsigned Alignment = MGT->getOriginalAlignment();
    1942             : 
    1943          33 :   SDValue MaskLo, MaskHi;
    1944          66 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1945             :     // Split Mask operand
    1946           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1947             :   else
    1948          33 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1949             : 
    1950          33 :   EVT MemoryVT = MGT->getMemoryVT();
    1951          33 :   EVT LoMemVT, HiMemVT;
    1952          33 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1953             : 
    1954          33 :   SDValue PassThruLo, PassThruHi;
    1955          66 :   if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
    1956           0 :     GetSplitVector(PassThru, PassThruLo, PassThruHi);
    1957             :   else
    1958          33 :     std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
    1959             : 
    1960          33 :   SDValue IndexHi, IndexLo;
    1961          66 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1962          33 :     GetSplitVector(Index, IndexLo, IndexHi);
    1963             :   else
    1964           0 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
    1965             : 
    1966          33 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1967          99 :     getMachineMemOperand(MGT->getPointerInfo(),
    1968             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1969          33 :                          Alignment, MGT->getAAInfo(), MGT->getRanges());
    1970             : 
    1971          33 :   SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
    1972          33 :   SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
    1973          33 :                                    OpsLo, MMO);
    1974             : 
    1975          33 :   MMO = DAG.getMachineFunction().
    1976          99 :     getMachineMemOperand(MGT->getPointerInfo(),
    1977             :                          MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),
    1978          33 :                          Alignment, MGT->getAAInfo(),
    1979             :                          MGT->getRanges());
    1980             : 
    1981          33 :   SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
    1982          33 :   SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
    1983          33 :                                    OpsHi, MMO);
    1984             : 
    1985             :   // Build a factor node to remember that this load is independent of the
    1986             :   // other one.
    1987          33 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1988          66 :                    Hi.getValue(1));
    1989             : 
    1990             :   // Legalize the chain result - switch anything that used the old chain to
    1991             :   // use the new one.
    1992          33 :   ReplaceValueWith(SDValue(MGT, 1), Ch);
    1993             : 
    1994          33 :   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
    1995          66 :                             Hi);
    1996          33 :   ReplaceValueWith(SDValue(MGT, 0), Res);
    1997          33 :   return SDValue();
    1998             : }
    1999             : 
    2000           8 : SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
    2001             :                                             unsigned OpNo) {
    2002           8 :   SDValue Ch  = N->getChain();
    2003           8 :   SDValue Ptr = N->getBasePtr();
    2004           8 :   SDValue Mask = N->getMask();
    2005           8 :   SDValue Data = N->getValue();
    2006           8 :   EVT MemoryVT = N->getMemoryVT();
    2007           8 :   unsigned Alignment = N->getOriginalAlignment();
    2008             :   SDLoc DL(N);
    2009             : 
    2010           8 :   EVT LoMemVT, HiMemVT;
    2011           8 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    2012             : 
    2013           8 :   SDValue DataLo, DataHi;
    2014          16 :   if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
    2015             :     // Split Data operand
    2016           8 :     GetSplitVector(Data, DataLo, DataHi);
    2017             :   else
    2018           0 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    2019             : 
    2020           8 :   SDValue MaskLo, MaskHi;
    2021          16 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    2022             :     // Split Mask operand
    2023           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    2024             :   else
    2025           8 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
    2026             : 
    2027             :   // if Alignment is equal to the vector size,
    2028             :   // take the half of it for the second part
    2029             :   unsigned SecondHalfAlignment =
    2030          16 :     (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
    2031             :        Alignment/2 : Alignment;
    2032             : 
    2033             :   SDValue Lo, Hi;
    2034           8 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    2035          24 :     getMachineMemOperand(N->getPointerInfo(),
    2036             :                          MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
    2037           8 :                          Alignment, N->getAAInfo(), N->getRanges());
    2038             : 
    2039           8 :   Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
    2040           8 :                           N->isTruncatingStore(),
    2041          16 :                           N->isCompressingStore());
    2042             : 
    2043           8 :   Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    2044           8 :                                    N->isCompressingStore());
    2045             :   unsigned HiOffset = LoMemVT.getStoreSize();
    2046             : 
    2047          24 :   MMO = DAG.getMachineFunction().getMachineMemOperand(
    2048             :       N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
    2049           8 :       HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
    2050             :       N->getRanges());
    2051             : 
    2052           8 :   Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
    2053          16 :                           N->isTruncatingStore(), N->isCompressingStore());
    2054             : 
    2055             :   // Build a factor node to remember that this store is independent of the
    2056             :   // other one.
    2057          16 :   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    2058             : }
    2059             : 
    2060          22 : SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
    2061             :                                               unsigned OpNo) {
    2062          22 :   SDValue Ch  = N->getChain();
    2063          22 :   SDValue Ptr = N->getBasePtr();
    2064          22 :   SDValue Mask = N->getMask();
    2065          22 :   SDValue Index = N->getIndex();
    2066          22 :   SDValue Scale = N->getScale();
    2067          22 :   SDValue Data = N->getValue();
    2068          22 :   EVT MemoryVT = N->getMemoryVT();
    2069          22 :   unsigned Alignment = N->getOriginalAlignment();
    2070             :   SDLoc DL(N);
    2071             : 
    2072             :   // Split all operands
    2073          22 :   EVT LoMemVT, HiMemVT;
    2074          22 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    2075             : 
    2076          22 :   SDValue DataLo, DataHi;
    2077          44 :   if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
    2078             :     // Split Data operand
    2079          13 :     GetSplitVector(Data, DataLo, DataHi);
    2080             :   else
    2081           9 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    2082             : 
    2083          22 :   SDValue MaskLo, MaskHi;
    2084          44 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    2085             :     // Split Mask operand
    2086           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    2087             :   else
    2088          22 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
    2089             : 
    2090          22 :   SDValue IndexHi, IndexLo;
    2091          44 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    2092          18 :     GetSplitVector(Index, IndexLo, IndexHi);
    2093             :   else
    2094           4 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
    2095             : 
    2096             :   SDValue Lo;
    2097          22 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    2098          66 :     getMachineMemOperand(N->getPointerInfo(),
    2099             :                          MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
    2100          22 :                          Alignment, N->getAAInfo(), N->getRanges());
    2101             : 
    2102          22 :   SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
    2103          22 :   Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
    2104          44 :                             DL, OpsLo, MMO);
    2105             : 
    2106          22 :   MMO = DAG.getMachineFunction().
    2107          66 :     getMachineMemOperand(N->getPointerInfo(),
    2108             :                          MachineMemOperand::MOStore,  HiMemVT.getStoreSize(),
    2109          22 :                          Alignment, N->getAAInfo(), N->getRanges());
    2110             : 
    2111             :   // The order of the Scatter operation after split is well defined. The "Hi"
    2112             :   // part comes after the "Lo". So these two operations should be chained one
    2113             :   // after another.
    2114          22 :   SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
    2115          22 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
    2116          44 :                               DL, OpsHi, MMO);
    2117             : }
    2118             : 
    2119        8785 : SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
    2120             :   assert(N->isUnindexed() && "Indexed store of vector?");
    2121             :   assert(OpNo == 1 && "Can only split the stored value");
    2122             :   SDLoc DL(N);
    2123             : 
    2124        8785 :   bool isTruncating = N->isTruncatingStore();
    2125        8785 :   SDValue Ch  = N->getChain();
    2126        8785 :   SDValue Ptr = N->getBasePtr();
    2127        8785 :   EVT MemoryVT = N->getMemoryVT();
    2128        8785 :   unsigned Alignment = N->getOriginalAlignment();
    2129        8785 :   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
    2130             :   AAMDNodes AAInfo = N->getAAInfo();
    2131        8785 :   SDValue Lo, Hi;
    2132        8785 :   GetSplitVector(N->getOperand(1), Lo, Hi);
    2133             : 
    2134        8785 :   EVT LoMemVT, HiMemVT;
    2135        8785 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    2136             : 
    2137             :   // Scalarize if the split halves are not byte-sized.
    2138        8785 :   if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
    2139         117 :     return TLI.scalarizeVectorStore(N, DAG);
    2140             : 
    2141        8668 :   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
    2142             : 
    2143        8668 :   if (isTruncating)
    2144           0 :     Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
    2145           0 :                            Alignment, MMOFlags, AAInfo);
    2146             :   else
    2147        8668 :     Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
    2148       17336 :                       AAInfo);
    2149             : 
    2150             :   // Increment the pointer to the other half.
    2151        8668 :   Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
    2152             : 
    2153        8668 :   if (isTruncating)
    2154           0 :     Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
    2155           0 :                            N->getPointerInfo().getWithOffset(IncrementSize),
    2156           0 :                            HiMemVT, Alignment, MMOFlags, AAInfo);
    2157             :   else
    2158        8668 :     Hi = DAG.getStore(Ch, DL, Hi, Ptr,
    2159        8668 :                       N->getPointerInfo().getWithOffset(IncrementSize),
    2160       17336 :                       Alignment, MMOFlags, AAInfo);
    2161             : 
    2162       17336 :   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    2163             : }
    2164             : 
    2165           2 : SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
    2166             :   SDLoc DL(N);
    2167             : 
    2168             :   // The input operands all must have the same type, and we know the result
    2169             :   // type is valid.  Convert this to a buildvector which extracts all the
    2170             :   // input elements.
    2171             :   // TODO: If the input elements are power-two vectors, we could convert this to
    2172             :   // a new CONCAT_VECTORS node with elements that are half-wide.
    2173             :   SmallVector<SDValue, 32> Elts;
    2174           4 :   EVT EltVT = N->getValueType(0).getVectorElementType();
    2175          10 :   for (const SDValue &Op : N->op_values()) {
    2176          24 :     for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
    2177          56 :          i != e; ++i) {
    2178          96 :       Elts.push_back(DAG.getNode(
    2179             :           ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
    2180          96 :           DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    2181             :     }
    2182             :   }
    2183             : 
    2184           6 :   return DAG.getBuildVector(N->getValueType(0), DL, Elts);
    2185             : }
    2186             : 
    2187        2060 : SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
    2188             :   // The result type is legal, but the input type is illegal.  If splitting
    2189             :   // ends up with the result type of each half still being legal, just
    2190             :   // do that.  If, however, that would result in an illegal result type,
    2191             :   // we can try to get more clever with power-two vectors. Specifically,
    2192             :   // split the input type, but also widen the result element size, then
    2193             :   // concatenate the halves and truncate again.  For example, consider a target
    2194             :   // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
    2195             :   // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
    2196             :   //   %inlo = v4i32 extract_subvector %in, 0
    2197             :   //   %inhi = v4i32 extract_subvector %in, 4
    2198             :   //   %lo16 = v4i16 trunc v4i32 %inlo
    2199             :   //   %hi16 = v4i16 trunc v4i32 %inhi
    2200             :   //   %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
    2201             :   //   %res = v8i8 trunc v8i16 %in16
    2202             :   //
    2203             :   // Without this transform, the original truncate would end up being
    2204             :   // scalarized, which is pretty much always a last resort.
    2205        2060 :   SDValue InVec = N->getOperand(0);
    2206        2060 :   EVT InVT = InVec->getValueType(0);
    2207        4120 :   EVT OutVT = N->getValueType(0);
    2208             :   unsigned NumElements = OutVT.getVectorNumElements();
    2209        2060 :   bool IsFloat = OutVT.isFloatingPoint();
    2210             : 
    2211             :   // Widening should have already made sure this is a power-two vector
    2212             :   // if we're trying to split it at all. assert() that's true, just in case.
    2213             :   assert(!(NumElements & 1) && "Splitting vector, but not in half!");
    2214             : 
    2215             :   unsigned InElementSize = InVT.getScalarSizeInBits();
    2216             :   unsigned OutElementSize = OutVT.getScalarSizeInBits();
    2217             : 
    2218             :   // If the input elements are only 1/2 the width of the result elements,
    2219             :   // just use the normal splitting. Our trick only work if there's room
    2220             :   // to split more than once.
    2221        2060 :   if (InElementSize <= OutElementSize * 2)
    2222        1386 :     return SplitVecOp_UnaryOp(N);
    2223             :   SDLoc DL(N);
    2224             : 
    2225             :   // Get the split input vector.
    2226         674 :   SDValue InLoVec, InHiVec;
    2227         674 :   GetSplitVector(InVec, InLoVec, InHiVec);
    2228             :   // Truncate them to 1/2 the element size.
    2229             :   EVT HalfElementVT = IsFloat ?
    2230          42 :     EVT::getFloatingPointVT(InElementSize/2) :
    2231         674 :     EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
    2232         674 :   EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
    2233         674 :                                 NumElements/2);
    2234        2022 :   SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
    2235        2022 :   SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
    2236             :   // Concatenate them to get the full intermediate truncation result.
    2237         674 :   EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
    2238         674 :   SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
    2239         674 :                                  HalfHi);
    2240             :   // Now finish up by truncating all the way down to the original result
    2241             :   // type. This should normally be something that ends up being legal directly,
    2242             :   // but in theory if a target has very wide vectors and an annoyingly
    2243             :   // restricted set of legal types, this split can chain to build things up.
    2244             :   return IsFloat
    2245          21 :              ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
    2246             :                            DAG.getTargetConstant(
    2247          42 :                                0, DL, TLI.getPointerTy(DAG.getDataLayout())))
    2248         695 :              : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
    2249             : }
    2250             : 
    2251         144 : SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
    2252             :   assert(N->getValueType(0).isVector() &&
    2253             :          N->getOperand(0).getValueType().isVector() &&
    2254             :          "Operand types must be vectors");
    2255             :   // The result has a legal vector type, but the input needs splitting.
    2256         144 :   SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
    2257             :   SDLoc DL(N);
    2258         288 :   GetSplitVector(N->getOperand(0), Lo0, Hi0);
    2259         288 :   GetSplitVector(N->getOperand(1), Lo1, Hi1);
    2260         432 :   unsigned PartElements = Lo0.getValueType().getVectorNumElements();
    2261         144 :   EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
    2262         288 :   EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
    2263             : 
    2264         432 :   LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
    2265         432 :   HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
    2266         288 :   SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
    2267         288 :   return PromoteTargetBoolean(Con, N->getValueType(0));
    2268             : }
    2269             : 
    2270             : 
    2271          82 : SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
    2272             :   // The result has a legal vector type, but the input needs splitting.
    2273          82 :   EVT ResVT = N->getValueType(0);
    2274          82 :   SDValue Lo, Hi;
    2275             :   SDLoc DL(N);
    2276         164 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    2277         164 :   EVT InVT = Lo.getValueType();
    2278             : 
    2279          82 :   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
    2280          82 :                                InVT.getVectorNumElements());
    2281             : 
    2282         246 :   Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
    2283         246 :   Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
    2284             : 
    2285         164 :   return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
    2286             : }
    2287             : 
    2288           8 : SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
    2289             :   // The result (and the first input) has a legal vector type, but the second
    2290             :   // input needs splitting.
    2291          24 :   return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
    2292             : }
    2293             : 
    2294             : 
    2295             : //===----------------------------------------------------------------------===//
    2296             : //  Result Vector Widening
    2297             : //===----------------------------------------------------------------------===//
    2298             : 
    2299       10459 : void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
    2300             :   LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
    2301             :              dbgs() << "\n");
    2302             : 
    2303             :   // See if the target wants to custom widen this node.
    2304       20918 :   if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
    2305         134 :     return;
    2306             : 
    2307       10325 :   SDValue Res = SDValue();
    2308       20650 :   switch (N->getOpcode()) {
    2309           0 :   default:
    2310             : #ifndef NDEBUG
    2311             :     dbgs() << "WidenVectorResult #" << ResNo << ": ";
    2312             :     N->dump(&DAG);
    2313             :     dbgs() << "\n";
    2314             : #endif
    2315           0 :     llvm_unreachable("Do not know how to widen the result of this operator!");
    2316             : 
    2317           0 :   case ISD::MERGE_VALUES:      Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
    2318        2099 :   case ISD::BITCAST:           Res = WidenVecRes_BITCAST(N); break;
    2319         644 :   case ISD::BUILD_VECTOR:      Res = WidenVecRes_BUILD_VECTOR(N); break;
    2320          49 :   case ISD::CONCAT_VECTORS:    Res = WidenVecRes_CONCAT_VECTORS(N); break;
    2321        1980 :   case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
    2322           0 :   case ISD::FP_ROUND_INREG:    Res = WidenVecRes_InregOp(N); break;
    2323          40 :   case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
    2324        2865 :   case ISD::LOAD:              Res = WidenVecRes_LOAD(N); break;
    2325           6 :   case ISD::SCALAR_TO_VECTOR:  Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
    2326          13 :   case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
    2327         119 :   case ISD::VSELECT:
    2328         119 :   case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
    2329           0 :   case ISD::SELECT_CC:         Res = WidenVecRes_SELECT_CC(N); break;
    2330         114 :   case ISD::SETCC:             Res = WidenVecRes_SETCC(N); break;
    2331         760 :   case ISD::UNDEF:             Res = WidenVecRes_UNDEF(N); break;
    2332             :   case ISD::VECTOR_SHUFFLE:
    2333         116 :     Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
    2334         116 :     break;
    2335             :   case ISD::MLOAD:
    2336          10 :     Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
    2337          10 :     break;
    2338             :   case ISD::MGATHER:
    2339          18 :     Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
    2340          18 :     break;
    2341             : 
    2342         222 :   case ISD::ADD:
    2343             :   case ISD::AND:
    2344             :   case ISD::MUL:
    2345             :   case ISD::MULHS:
    2346             :   case ISD::MULHU:
    2347             :   case ISD::OR:
    2348             :   case ISD::SUB:
    2349             :   case ISD::XOR:
    2350             :   case ISD::FMINNUM:
    2351             :   case ISD::FMAXNUM:
    2352             :   case ISD::FMINNAN:
    2353             :   case ISD::FMAXNAN:
    2354             :   case ISD::SMIN:
    2355             :   case ISD::SMAX:
    2356             :   case ISD::UMIN:
    2357             :   case ISD::UMAX:
    2358         222 :     Res = WidenVecRes_Binary(N);
    2359         222 :     break;
    2360             : 
    2361         179 :   case ISD::FADD:
    2362             :   case ISD::FMUL:
    2363             :   case ISD::FPOW:
    2364             :   case ISD::FSUB:
    2365             :   case ISD::FDIV:
    2366             :   case ISD::FREM:
    2367             :   case ISD::SDIV:
    2368             :   case ISD::UDIV:
    2369             :   case ISD::SREM:
    2370             :   case ISD::UREM:
    2371         179 :     Res = WidenVecRes_BinaryCanTrap(N);
    2372         179 :     break;
    2373             : 
    2374          72 :   case ISD::STRICT_FADD:
    2375             :   case ISD::STRICT_FSUB:
    2376             :   case ISD::STRICT_FMUL:
    2377             :   case ISD::STRICT_FDIV:
    2378             :   case ISD::STRICT_FREM:
    2379             :   case ISD::STRICT_FSQRT:
    2380             :   case ISD::STRICT_FMA:
    2381             :   case ISD::STRICT_FPOW:
    2382             :   case ISD::STRICT_FPOWI:
    2383             :   case ISD::STRICT_FSIN:
    2384             :   case ISD::STRICT_FCOS:
    2385             :   case ISD::STRICT_FEXP:
    2386             :   case ISD::STRICT_FEXP2:
    2387             :   case ISD::STRICT_FLOG:
    2388             :   case ISD::STRICT_FLOG10:
    2389             :   case ISD::STRICT_FLOG2:
    2390             :   case ISD::STRICT_FRINT:
    2391             :   case ISD::STRICT_FNEARBYINT:
    2392          72 :     Res = WidenVecRes_StrictFP(N);
    2393          72 :     break;
    2394             : 
    2395           5 :   case ISD::FCOPYSIGN:
    2396           5 :     Res = WidenVecRes_FCOPYSIGN(N);
    2397           5 :     break;
    2398             : 
    2399           1 :   case ISD::FPOWI:
    2400           1 :     Res = WidenVecRes_POWI(N);
    2401           1 :     break;
    2402             : 
    2403          37 :   case ISD::SHL:
    2404             :   case ISD::SRA:
    2405             :   case ISD::SRL:
    2406          37 :     Res = WidenVecRes_Shift(N);
    2407          37 :     break;
    2408             : 
    2409          14 :   case ISD::ANY_EXTEND_VECTOR_INREG:
    2410             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
    2411             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
    2412          14 :     Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
    2413          14 :     break;
    2414             : 
    2415         904 :   case ISD::ANY_EXTEND:
    2416             :   case ISD::FP_EXTEND:
    2417             :   case ISD::FP_ROUND:
    2418             :   case ISD::FP_TO_SINT:
    2419             :   case ISD::FP_TO_UINT:
    2420             :   case ISD::SIGN_EXTEND:
    2421             :   case ISD::SINT_TO_FP:
    2422             :   case ISD::TRUNCATE:
    2423             :   case ISD::UINT_TO_FP:
    2424             :   case ISD::ZERO_EXTEND:
    2425         904 :     Res = WidenVecRes_Convert(N);
    2426         904 :     break;
    2427             : 
    2428          51 :   case ISD::FABS:
    2429             :   case ISD::FCEIL:
    2430             :   case ISD::FCOS:
    2431             :   case ISD::FEXP:
    2432             :   case ISD::FEXP2:
    2433             :   case ISD::FFLOOR:
    2434             :   case ISD::FLOG:
    2435             :   case ISD::FLOG10:
    2436             :   case ISD::FLOG2:
    2437             :   case ISD::FNEARBYINT:
    2438             :   case ISD::FRINT:
    2439             :   case ISD::FROUND:
    2440             :   case ISD::FSIN:
    2441             :   case ISD::FSQRT:
    2442             :   case ISD::FTRUNC: {
    2443             :     // We're going to widen this vector op to a legal type by padding with undef
    2444             :     // elements. If the wide vector op is eventually going to be expanded to
    2445             :     // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
    2446             :     // libcalls on the undef elements. We are assuming that if the scalar op
    2447             :     // requires expanding, then the vector op needs expanding too.
    2448          51 :     EVT VT = N->getValueType(0);
    2449          51 :     if (TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
    2450          24 :       EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2451             :       assert(!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
    2452             :              "Target supports vector op, but scalar requires expansion?");
    2453          48 :       Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
    2454             :       break;
    2455          51 :     }
    2456             :   }
    2457             :   // If the target has custom/legal support for the scalar FP intrinsic ops
    2458             :   // (they are probably not destined to become libcalls), then widen those like
    2459             :   // any other unary ops.
    2460             :   LLVM_FALLTHROUGH;
    2461             : 
    2462             :   case ISD::BITREVERSE:
    2463             :   case ISD::BSWAP:
    2464             :   case ISD::CTLZ:
    2465             :   case ISD::CTPOP:
    2466             :   case ISD::CTTZ:
    2467             :   case ISD::FNEG:
    2468             :   case ISD::FCANONICALIZE:
    2469          32 :     Res = WidenVecRes_Unary(N);
    2470          32 :     break;
    2471           2 :   case ISD::FMA:
    2472           2 :     Res = WidenVecRes_Ternary(N);
    2473           2 :     break;
    2474             :   }
    2475             : 
    2476             :   // If Res is null, the sub-method took care of registering the result.
    2477       10325 :   if (Res.getNode())
    2478       10325 :     SetWidenedVector(SDValue(N, ResNo), Res);
    2479             : }
    2480             : 
    2481           2 : SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
    2482             :   // Ternary op widening.
    2483             :   SDLoc dl(N);
    2484           4 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2485           4 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2486           4 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2487           4 :   SDValue InOp3 = GetWidenedVector(N->getOperand(2));
    2488           6 :   return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
    2489             : }
    2490             : 
    2491         222 : SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
    2492             :   // Binary op widening.
    2493             :   SDLoc dl(N);
    2494         444 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2495         444 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2496         444 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2497         666 :   return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
    2498             : }
    2499             : 
    2500             : // Given a vector of operations that have been broken up to widen, see
    2501             : // if we can collect them together into the next widest legal VT. This
    2502             : // implementation is trap-safe.
    2503         113 : static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
    2504             :                                  SmallVectorImpl<SDValue> &ConcatOps,
    2505             :                                  unsigned ConcatEnd, EVT VT, EVT MaxVT,
    2506             :                                  EVT WidenVT) {
    2507             :   // Check to see if we have a single operation with the widen type.
    2508         113 :   if (ConcatEnd == 1) {
    2509          12 :     VT = ConcatOps[0].getValueType();
    2510          12 :     if (VT == WidenVT)
    2511           0 :       return ConcatOps[0];
    2512             :   }
    2513             : 
    2514             :   SDLoc dl(ConcatOps[0]);
    2515         113 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    2516             :   int Idx = 0;
    2517             : 
    2518             :   // while (Some element of ConcatOps is not of type MaxVT) {
    2519             :   //   From the end of ConcatOps, collect elements of the same type and put
    2520             :   //   them into an op of the next larger supported type
    2521             :   // }
    2522         474 :   while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
    2523         135 :     Idx = ConcatEnd - 1;
    2524         405 :     VT = ConcatOps[Idx--].getValueType();
    2525         253 :     while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
    2526         118 :       Idx--;
    2527             : 
    2528         157 :     int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
    2529             :     EVT NextVT;
    2530             :     do {
    2531         211 :       NextSize *= 2;
    2532         211 :       NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
    2533             :     } while (!TLI.isTypeLegal(NextVT));
    2534             : 
    2535         135 :     if (!VT.isVector()) {
    2536             :       // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
    2537         113 :       SDValue VecOp = DAG.getUNDEF(NextVT);
    2538         113 :       unsigned NumToInsert = ConcatEnd - Idx - 1;
    2539         326 :       for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
    2540         213 :         VecOp = DAG.getNode(
    2541         213 :             ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
    2542         213 :             DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2543             :       }
    2544         113 :       ConcatOps[Idx+1] = VecOp;
    2545         113 :       ConcatEnd = Idx + 2;
    2546             :     } else {
    2547             :       // Vector type, create a CONCAT_VECTORS of type NextVT
    2548          22 :       SDValue undefVec = DAG.getUNDEF(VT);
    2549          22 :       unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
    2550          22 :       SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
    2551          22 :       unsigned RealVals = ConcatEnd - Idx - 1;
    2552             :       unsigned SubConcatEnd = 0;
    2553          22 :       unsigned SubConcatIdx = Idx + 1;
    2554          62 :       while (SubConcatEnd < RealVals)
    2555         120 :         SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
    2556          26 :       while (SubConcatEnd < OpsToConcat)
    2557           8 :         SubConcatOps[SubConcatEnd++] = undefVec;
    2558          22 :       ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
    2559          22 :                                             NextVT, SubConcatOps);
    2560          22 :       ConcatEnd = SubConcatIdx + 1;
    2561             :     }
    2562             :   }
    2563             : 
    2564             :   // Check to see if we have a single operation with the widen type.
    2565         113 :   if (ConcatEnd == 1) {
    2566          86 :     VT = ConcatOps[0].getValueType();
    2567          86 :     if (VT == WidenVT)
    2568          86 :       return ConcatOps[0];
    2569             :   }
    2570             : 
    2571             :   // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
    2572          27 :   unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
    2573          27 :   if (NumOps != ConcatEnd ) {
    2574           3 :     SDValue UndefVal = DAG.getUNDEF(MaxVT);
    2575           6 :     for (unsigned j = ConcatEnd; j < NumOps; ++j)
    2576           6 :       ConcatOps[j] = UndefVal;
    2577             :   }
    2578             :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
    2579          27 :                      makeArrayRef(ConcatOps.data(), NumOps));
    2580             : }
    2581             : 
    2582         183 : SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
    2583             :   // Binary op widening for operations that can trap.
    2584         183 :   unsigned Opcode = N->getOpcode();
    2585             :   SDLoc dl(N);
    2586         366 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2587         183 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    2588         183 :   EVT VT = WidenVT;
    2589             :   unsigned NumElts =  VT.getVectorNumElements();
    2590         183 :   const SDNodeFlags Flags = N->getFlags();
    2591         239 :   while (!TLI.isTypeLegal(VT) && NumElts != 1) {
    2592          27 :     NumElts = NumElts / 2;
    2593          27 :     VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2594             :   }
    2595             : 
    2596         183 :   if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
    2597             :     // Operation doesn't trap so just widen as normal.
    2598         276 :     SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2599         276 :     SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2600         276 :     return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
    2601             :   }
    2602             : 
    2603             :   // No legal vector version so unroll the vector operation and then widen.
    2604          45 :   if (NumElts == 1)
    2605           8 :     return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
    2606             : 
    2607             :   // Since the operation can trap, apply operation on the original vector.
    2608          41 :   EVT MaxVT = VT;
    2609          82 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2610          82 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2611          82 :   unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
    2612             : 
    2613          41 :   SmallVector<SDValue, 16> ConcatOps(CurNumElts);
    2614             :   unsigned ConcatEnd = 0;  // Current ConcatOps index.
    2615             :   int Idx = 0;        // Current Idx into input vectors.
    2616             : 
    2617             :   // NumElts := greatest legal vector size (at most WidenVT)
    2618             :   // while (orig. vector has unhandled elements) {
    2619             :   //   take munches of size NumElts from the beginning and add to ConcatOps
    2620             :   //   NumElts := next smaller supported vector size or 1
    2621             :   // }
    2622          86 :   while (CurNumElts != 0) {
    2623          57 :     while (CurNumElts >= NumElts) {
    2624          12 :       SDValue EOp1 = DAG.getNode(
    2625             :           ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
    2626          12 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2627          12 :       SDValue EOp2 = DAG.getNode(
    2628             :           ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
    2629          12 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2630          12 :       ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
    2631          12 :       Idx += NumElts;
    2632          12 :       CurNumElts -= NumElts;
    2633             :     }
    2634             :     do {
    2635          85 :       NumElts = NumElts / 2;
    2636          85 :       VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2637         166 :     } while (!TLI.isTypeLegal(VT) && NumElts != 1);
    2638             : 
    2639          45 :     if (NumElts == 1) {
    2640         110 :       for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
    2641          69 :         SDValue EOp1 = DAG.getNode(
    2642             :             ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
    2643          69 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2644          69 :         SDValue EOp2 = DAG.getNode(
    2645             :             ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
    2646          69 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2647         138 :         ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
    2648          69 :                                              EOp1, EOp2, Flags);
    2649             :       }
    2650             :       CurNumElts = 0;
    2651             :     }
    2652             :   }
    2653             : 
    2654          41 :   return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
    2655             : }
    2656             : 
    2657          72 : SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
    2658             :   // StrictFP op widening for operations that can trap.
    2659          72 :   unsigned NumOpers = N->getNumOperands();
    2660          72 :   unsigned Opcode = N->getOpcode();
    2661             :   SDLoc dl(N);
    2662         144 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2663          72 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    2664          72 :   EVT VT = WidenVT;
    2665             :   unsigned NumElts = VT.getVectorNumElements();
    2666         108 :   while (!TLI.isTypeLegal(VT) && NumElts != 1) {
    2667          18 :     NumElts = NumElts / 2;
    2668          18 :     VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2669             :   }
    2670             : 
    2671             :   // No legal vector version so unroll the vector operation and then widen.
    2672          72 :   if (NumElts == 1)
    2673           0 :     return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
    2674             : 
    2675             :   // Since the operation can trap, apply operation on the original vector.
    2676          72 :   EVT MaxVT = VT;
    2677             :   SmallVector<SDValue, 4> InOps;
    2678         144 :   unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
    2679             : 
    2680          72 :   SmallVector<SDValue, 16> ConcatOps(CurNumElts);
    2681             :   SmallVector<SDValue, 16> Chains;
    2682             :   unsigned ConcatEnd = 0;  // Current ConcatOps index.
    2683             :   int Idx = 0;        // Current Idx into input vectors.
    2684             : 
    2685             :   // The Chain is the first operand.
    2686         144 :   InOps.push_back(N->getOperand(0));
    2687             : 
    2688             :   // Now process the remaining operands. 
    2689         180 :   for (unsigned i = 1; i < NumOpers; ++i) {
    2690         108 :     SDValue Oper = N->getOperand(i);
    2691             : 
    2692         324 :     if (Oper.getValueType().isVector()) {
    2693             :       assert(Oper.getValueType() == N->getValueType(0) && 
    2694             :              "Invalid operand type to widen!");
    2695         104 :       Oper = GetWidenedVector(Oper);
    2696             :     }
    2697             : 
    2698         108 :     InOps.push_back(Oper);
    2699             :   }
    2700             : 
    2701             :   // NumElts := greatest legal vector size (at most WidenVT)
    2702             :   // while (orig. vector has unhandled elements) {
    2703             :   //   take munches of size NumElts from the beginning and add to ConcatOps
    2704             :   //   NumElts := next smaller supported vector size or 1
    2705             :   // }
    2706         162 :   while (CurNumElts != 0) {
    2707         126 :     while (CurNumElts >= NumElts) {
    2708             :       SmallVector<SDValue, 4> EOps;
    2709             :       
    2710         126 :       for (unsigned i = 0; i < NumOpers; ++i) {
    2711          90 :         SDValue Op = InOps[i];
    2712             :         
    2713         270 :         if (Op.getValueType().isVector()) 
    2714          52 :           Op = DAG.getNode(
    2715             :             ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
    2716          52 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2717             : 
    2718          90 :         EOps.push_back(Op);
    2719             :       }
    2720             : 
    2721          36 :       EVT OperVT[] = {VT, MVT::Other};
    2722          72 :       SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
    2723          36 :       ConcatOps[ConcatEnd++] = Oper;
    2724          36 :       Chains.push_back(Oper.getValue(1));
    2725          36 :       Idx += NumElts;
    2726          36 :       CurNumElts -= NumElts;
    2727             :     }
    2728             :     do {
    2729         126 :       NumElts = NumElts / 2;
    2730         126 :       VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2731         234 :     } while (!TLI.isTypeLegal(VT) && NumElts != 1);
    2732             : 
    2733          90 :     if (NumElts == 1) {
    2734         216 :       for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
    2735             :         SmallVector<SDValue, 4> EOps;
    2736             : 
    2737         504 :         for (unsigned i = 0; i < NumOpers; ++i) {
    2738         360 :           SDValue Op = InOps[i];
    2739             : 
    2740        1080 :           if (Op.getValueType().isVector())
    2741         208 :             Op = DAG.getNode(
    2742             :               ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
    2743             :               DAG.getConstant(Idx, dl, 
    2744         208 :                               TLI.getVectorIdxTy(DAG.getDataLayout())));
    2745             : 
    2746         360 :           EOps.push_back(Op);
    2747             :         }
    2748             : 
    2749         144 :         EVT WidenVT[] = {WidenEltVT, MVT::Other}; 
    2750         288 :         SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
    2751         144 :         ConcatOps[ConcatEnd++] = Oper;
    2752         144 :         Chains.push_back(Oper.getValue(1));
    2753             :       }
    2754             :       CurNumElts = 0;
    2755             :     }
    2756             :   }
    2757             : 
    2758             :   // Build a factor node to remember all the Ops that have been created.
    2759          72 :   SDValue NewChain;
    2760         144 :   if (Chains.size() == 1)
    2761           0 :     NewChain = Chains[0];
    2762             :   else
    2763         144 :     NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
    2764          72 :   ReplaceValueWith(SDValue(N, 1), NewChain);
    2765             : 
    2766          72 :   return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
    2767             : }
    2768             : 
    2769         904 : SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
    2770         904 :   SDValue InOp = N->getOperand(0);
    2771             :   SDLoc DL(N);
    2772             : 
    2773        1808 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2774         904 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2775             : 
    2776         904 :   EVT InVT = InOp.getValueType();
    2777         904 :   EVT InEltVT = InVT.getVectorElementType();
    2778         904 :   EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
    2779             : 
    2780         904 :   unsigned Opcode = N->getOpcode();
    2781         904 :   unsigned InVTNumElts = InVT.getVectorNumElements();
    2782         904 :   const SDNodeFlags Flags = N->getFlags();
    2783         904 :   if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
    2784         516 :     InOp = GetWidenedVector(N->getOperand(0));
    2785         258 :     InVT = InOp.getValueType();
    2786         258 :     InVTNumElts = InVT.getVectorNumElements();
    2787         258 :     if (InVTNumElts == WidenNumElts) {
    2788         245 :       if (N->getNumOperands() == 1)
    2789         466 :         return DAG.getNode(Opcode, DL, WidenVT, InOp);
    2790          24 :       return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
    2791             :     }
    2792          13 :     if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
    2793             :       // If both input and result vector types are of same width, extend
    2794             :       // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
    2795             :       // accepts fewer elements in the result than in the input.
    2796          13 :       if (Opcode == ISD::SIGN_EXTEND)
    2797           4 :         return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
    2798           9 :       if (Opcode == ISD::ZERO_EXTEND)
    2799           5 :         return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
    2800             :     }
    2801             :   }
    2802             : 
    2803         650 :   if (TLI.isTypeLegal(InWidenVT)) {
    2804             :     // Because the result and the input are different vector types, widening
    2805             :     // the result could create a legal type but widening the input might make
    2806             :     // it an illegal type that might lead to repeatedly splitting the input
    2807             :     // and then widening it. To avoid this, we widen the input only if
    2808             :     // it results in a legal type.
    2809          60 :     if (WidenNumElts % InVTNumElts == 0) {
    2810             :       // Widen the input and call convert on the widened input vector.
    2811          60 :       unsigned NumConcat = WidenNumElts/InVTNumElts;
    2812          60 :       SmallVector<SDValue, 16> Ops(NumConcat);
    2813          60 :       Ops[0] = InOp;
    2814          60 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    2815         120 :       for (unsigned i = 1; i != NumConcat; ++i)
    2816         120 :         Ops[i] = UndefVal;
    2817         120 :       SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
    2818          60 :       if (N->getNumOperands() == 1)
    2819         120 :         return DAG.getNode(Opcode, DL, WidenVT, InVec);
    2820           0 :       return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
    2821             :     }
    2822             : 
    2823           0 :     if (InVTNumElts % WidenNumElts == 0) {
    2824           0 :       SDValue InVal = DAG.getNode(
    2825             :           ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
    2826           0 :           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2827             :       // Extract the input and convert the shorten input vector.
    2828           0 :       if (N->getNumOperands() == 1)
    2829           0 :         return DAG.getNode(Opcode, DL, WidenVT, InVal);
    2830           0 :       return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
    2831             :     }
    2832             :   }
    2833             : 
    2834             :   // Otherwise unroll into some nasty scalar code and rebuild the vector.
    2835         590 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    2836         590 :   EVT EltVT = WidenVT.getVectorElementType();
    2837         590 :   unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
    2838             :   unsigned i;
    2839        2234 :   for (i=0; i < MinElts; ++i) {
    2840        1644 :     SDValue Val = DAG.getNode(
    2841             :         ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
    2842        1644 :         DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2843        1644 :     if (N->getNumOperands() == 1)
    2844        3224 :       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
    2845             :     else
    2846          64 :       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
    2847             :   }
    2848             : 
    2849         590 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    2850        2306 :   for (; i < WidenNumElts; ++i)
    2851        3432 :     Ops[i] = UndefVal;
    2852             : 
    2853        1180 :   return DAG.getBuildVector(WidenVT, DL, Ops);
    2854             : }
    2855             : 
    2856          14 : SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
    2857          14 :   unsigned Opcode = N->getOpcode();
    2858          14 :   SDValue InOp = N->getOperand(0);
    2859             :   SDLoc DL(N);
    2860             : 
    2861          28 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2862          14 :   EVT WidenSVT = WidenVT.getVectorElementType();
    2863          14 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2864             : 
    2865          14 :   EVT InVT = InOp.getValueType();
    2866          14 :   EVT InSVT = InVT.getVectorElementType();
    2867          14 :   unsigned InVTNumElts = InVT.getVectorNumElements();
    2868             : 
    2869          14 :   if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
    2870          14 :     InOp = GetWidenedVector(InOp);
    2871          14 :     InVT = InOp.getValueType();
    2872          14 :     if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
    2873          14 :       switch (Opcode) {
    2874           0 :       case ISD::ANY_EXTEND_VECTOR_INREG:
    2875           0 :         return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
    2876           5 :       case ISD::SIGN_EXTEND_VECTOR_INREG:
    2877           5 :         return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
    2878           9 :       case ISD::ZERO_EXTEND_VECTOR_INREG:
    2879           9 :         return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
    2880             :       }
    2881             :     }
    2882             :   }
    2883             : 
    2884             :   // Unroll, extend the scalars and rebuild the vector.
    2885             :   SmallVector<SDValue, 16> Ops;
    2886           0 :   for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
    2887           0 :     SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
    2888           0 :       DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2889           0 :     switch (Opcode) {
    2890           0 :     case ISD::ANY_EXTEND_VECTOR_INREG:
    2891           0 :       Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
    2892           0 :       break;
    2893           0 :     case ISD::SIGN_EXTEND_VECTOR_INREG:
    2894           0 :       Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
    2895           0 :       break;
    2896           0 :     case ISD::ZERO_EXTEND_VECTOR_INREG:
    2897           0 :       Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
    2898           0 :       break;
    2899           0 :     default:
    2900           0 :       llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
    2901             :     }
    2902           0 :     Ops.push_back(Val);
    2903             :   }
    2904             : 
    2905           0 :   while (Ops.size() != WidenNumElts)
    2906           0 :     Ops.push_back(DAG.getUNDEF(WidenSVT));
    2907             : 
    2908           0 :   return DAG.getBuildVector(WidenVT, DL, Ops);
    2909             : }
    2910             : 
    2911           5 : SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
    2912             :   // If this is an FCOPYSIGN with same input types, we can treat it as a
    2913             :   // normal (can trap) binary op.
    2914          13 :   if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
    2915           4 :     return WidenVecRes_BinaryCanTrap(N);
    2916             : 
    2917             :   // If the types are different, fall back to unrolling.
    2918           2 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2919           2 :   return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
    2920             : }
    2921             : 
    2922           1 : SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
    2923           2 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2924           2 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2925           1 :   SDValue ShOp = N->getOperand(1);
    2926           1 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
    2927             : }
    2928             : 
    2929          37 : SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
    2930          74 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2931          74 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2932          37 :   SDValue ShOp = N->getOperand(1);
    2933             : 
    2934          37 :   EVT ShVT = ShOp.getValueType();
    2935          37 :   if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
    2936          37 :     ShOp = GetWidenedVector(ShOp);
    2937          37 :     ShVT = ShOp.getValueType();
    2938             :   }
    2939          37 :   EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
    2940             :                                    ShVT.getVectorElementType(),
    2941          37 :                                    WidenVT.getVectorNumElements());
    2942          37 :   if (ShVT != ShWidenVT)
    2943           0 :     ShOp = ModifyToType(ShOp, ShWidenVT);
    2944             : 
    2945          37 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
    2946             : }
    2947             : 
    2948          32 : SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
    2949             :   // Unary op widening.
    2950          64 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2951          64 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2952          32 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
    2953             : }
    2954             : 
    2955          13 : SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
    2956          26 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2957          13 :   EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
    2958          13 :                                cast<VTSDNode>(N->getOperand(1))->getVT()
    2959             :                                  .getVectorElementType(),
    2960          13 :                                WidenVT.getVectorNumElements());
    2961          26 :   SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
    2962          13 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
    2963          39 :                      WidenVT, WidenLHS, DAG.getValueType(ExtVT));
    2964             : }
    2965             : 
    2966           0 : SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
    2967           0 :   SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
    2968           0 :   return GetWidenedVector(WidenVec);
    2969             : }
    2970             : 
    2971        2099 : SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
    2972        2099 :   SDValue InOp = N->getOperand(0);
    2973        2099 :   EVT InVT = InOp.getValueType();
    2974        2099 :   EVT VT = N->getValueType(0);
    2975        2099 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2976             :   SDLoc dl(N);
    2977             : 
    2978        2099 :   switch (getTypeAction(InVT)) {
    2979             :   case TargetLowering::TypeLegal:
    2980             :     break;
    2981             :   case TargetLowering::TypePromoteInteger:
    2982             :     // If the incoming type is a vector that is being promoted, then
    2983             :     // we know that the elements are arranged differently and that we
    2984             :     // must perform the conversion using a stack slot.
    2985          39 :     if (InVT.isVector())
    2986             :       break;
    2987             : 
    2988             :     // If the InOp is promoted to the same size, convert it.  Otherwise,
    2989             :     // fall out of the switch and widen the promoted input.
    2990          22 :     InOp = GetPromotedInteger(InOp);
    2991          22 :     InVT = InOp.getValueType();
    2992          22 :     if (WidenVT.bitsEq(InVT))
    2993          28 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
    2994             :     break;
    2995             :   case TargetLowering::TypeSoftenFloat:
    2996             :   case TargetLowering::TypePromoteFloat:
    2997             :   case TargetLowering::TypeExpandInteger:
    2998             :   case TargetLowering::TypeExpandFloat:
    2999             :   case TargetLowering::TypeScalarizeVector:
    3000             :   case TargetLowering::TypeSplitVector:
    3001             :     break;
    3002        1980 :   case TargetLowering::TypeWidenVector:
    3003             :     // If the InOp is widened to the same size, convert it.  Otherwise, fall
    3004             :     // out of the switch and widen the widened input.
    3005        1980 :     InOp = GetWidenedVector(InOp);
    3006        1980 :     InVT = InOp.getValueType();
    3007        1980 :     if (WidenVT.bitsEq(InVT))
    3008             :       // The input widens to the same size. Convert to the widen value.
    3009        3960 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
    3010             :     break;
    3011             :   }
    3012             : 
    3013         105 :   unsigned WidenSize = WidenVT.getSizeInBits();
    3014         105 :   unsigned InSize = InVT.getSizeInBits();
    3015             :   // x86mmx is not an acceptable vector element type, so don't try.
    3016         105 :   if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
    3017             :     // Determine new input vector type.  The new input vector type will use
    3018             :     // the same element type (if its a vector) or use the input type as a
    3019             :     // vector.  It is the same size as the type to widen to.
    3020             :     EVT NewInVT;
    3021          80 :     unsigned NewNumElts = WidenSize / InSize;
    3022          80 :     if (InVT.isVector()) {
    3023          17 :       EVT InEltVT = InVT.getVectorElementType();
    3024          17 :       NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
    3025          17 :                                  WidenSize / InEltVT.getSizeInBits());
    3026             :     } else {
    3027          63 :       NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
    3028             :     }
    3029             : 
    3030          80 :     if (TLI.isTypeLegal(NewInVT)) {
    3031             :       // Because the result and the input are different vector types, widening
    3032             :       // the result could create a legal type but widening the input might make
    3033             :       // it an illegal type that might lead to repeatedly splitting the input
    3034             :       // and then widening it. To avoid this, we widen the input only if
    3035             :       // it results in a legal type.
    3036          78 :       SmallVector<SDValue, 16> Ops(NewNumElts);
    3037          78 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    3038          78 :       Ops[0] = InOp;
    3039         204 :       for (unsigned i = 1; i < NewNumElts; ++i)
    3040         252 :         Ops[i] = UndefVal;
    3041             : 
    3042          78 :       SDValue NewVec;
    3043          78 :       if (InVT.isVector())
    3044          34 :         NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
    3045             :       else
    3046         122 :         NewVec = DAG.getBuildVector(NewInVT, dl, Ops);
    3047         156 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
    3048             :     }
    3049             :   }
    3050             : 
    3051          27 :   return CreateStackStoreLoad(InOp, WidenVT);
    3052             : }
    3053             : 
    3054         644 : SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
    3055             :   SDLoc dl(N);
    3056             :   // Build a vector with undefined for the new nodes.
    3057         644 :   EVT VT = N->getValueType(0);
    3058             : 
    3059             :   // Integer BUILD_VECTOR operands may be larger than the node's vector element
    3060             :   // type. The UNDEFs need to have the same type as the existing operands.
    3061        1288 :   EVT EltVT = N->getOperand(0).getValueType();
    3062             :   unsigned NumElts = VT.getVectorNumElements();
    3063             : 
    3064         644 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3065             :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3066             : 
    3067         644 :   SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
    3068             :   assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
    3069         644 :   NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
    3070             : 
    3071        1288 :   return DAG.getBuildVector(WidenVT, dl, NewOps);
    3072             : }
    3073             : 
    3074          49 : SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
    3075          49 :   EVT InVT = N->getOperand(0).getValueType();
    3076          98 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3077             :   SDLoc dl(N);
    3078             :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3079             :   unsigned NumInElts = InVT.getVectorNumElements();
    3080          49 :   unsigned NumOperands = N->getNumOperands();
    3081             : 
    3082             :   bool InputWidened = false; // Indicates we need to widen the input.
    3083          49 :   if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
    3084          16 :     if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
    3085             :       // Add undef vectors to widen to correct length.
    3086             :       unsigned NumConcat = WidenVT.getVectorNumElements() /
    3087          16 :                            InVT.getVectorNumElements();
    3088          16 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    3089          16 :       SmallVector<SDValue, 16> Ops(NumConcat);
    3090          80 :       for (unsigned i=0; i < NumOperands; ++i)
    3091         128 :         Ops[i] = N->getOperand(i);
    3092          48 :       for (unsigned i = NumOperands; i != NumConcat; ++i)
    3093          64 :         Ops[i] = UndefVal;
    3094          32 :       return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
    3095             :     }
    3096             :   } else {
    3097             :     InputWidened = true;
    3098          33 :     if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
    3099             :       // The inputs and the result are widen to the same value.
    3100             :       unsigned i;
    3101          12 :       for (i=1; i < NumOperands; ++i)
    3102          24 :         if (!N->getOperand(i).isUndef())
    3103             :           break;
    3104             : 
    3105           8 :       if (i == NumOperands)
    3106             :         // Everything but the first operand is an UNDEF so just return the
    3107             :         // widened first operand.
    3108           8 :         return GetWidenedVector(N->getOperand(0));
    3109             : 
    3110           4 :       if (NumOperands == 2) {
    3111             :         // Replace concat of two operands with a shuffle.
    3112           4 :         SmallVector<int, 16> MaskOps(WidenNumElts, -1);
    3113          16 :         for (unsigned i = 0; i < NumInElts; ++i) {
    3114          12 :           MaskOps[i] = i;
    3115          24 :           MaskOps[i + NumInElts] = i + WidenNumElts;
    3116             :         }
    3117           4 :         return DAG.getVectorShuffle(WidenVT, dl,
    3118           4 :                                     GetWidenedVector(N->getOperand(0)),
    3119           4 :                                     GetWidenedVector(N->getOperand(1)),
    3120           4 :                                     MaskOps);
    3121             :       }
    3122             :     }
    3123             :   }
    3124             : 
    3125             :   // Fall back to use extracts and build vector.
    3126          25 :   EVT EltVT = WidenVT.getVectorElementType();
    3127          25 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    3128             :   unsigned Idx = 0;
    3129          91 :   for (unsigned i=0; i < NumOperands; ++i) {
    3130          66 :     SDValue InOp = N->getOperand(i);
    3131          66 :     if (InputWidened)
    3132          66 :       InOp = GetWidenedVector(InOp);
    3133         644 :     for (unsigned j=0; j < NumInElts; ++j)
    3134        1156 :       Ops[Idx++] = DAG.getNode(
    3135             :           ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    3136         578 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3137             :   }
    3138          25 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    3139         231 :   for (; Idx < WidenNumElts; ++Idx)
    3140         412 :     Ops[Idx] = UndefVal;
    3141          50 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    3142             : }
    3143             : 
    3144        1980 : SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
    3145        1980 :   EVT      VT = N->getValueType(0);
    3146        1980 :   EVT      WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3147             :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3148        1980 :   SDValue  InOp = N->getOperand(0);
    3149        1980 :   SDValue  Idx  = N->getOperand(1);
    3150             :   SDLoc dl(N);
    3151             : 
    3152        1980 :   if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
    3153          56 :     InOp = GetWidenedVector(InOp);
    3154             : 
    3155        1980 :   EVT InVT = InOp.getValueType();
    3156             : 
    3157             :   // Check if we can just return the input vector after widening.
    3158        1980 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    3159        1980 :   if (IdxVal == 0 && InVT == WidenVT)
    3160        1713 :     return InOp;
    3161             : 
    3162             :   // Check if we can extract from the vector.
    3163             :   unsigned InNumElts = InVT.getVectorNumElements();
    3164         267 :   if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
    3165          12 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
    3166             : 
    3167             :   // We could try widening the input to the right length but for now, extract
    3168             :   // the original elements, fill the rest with undefs and build a vector.
    3169         261 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    3170         261 :   EVT EltVT = VT.getVectorElementType();
    3171             :   unsigned NumElts = VT.getVectorNumElements();
    3172             :   unsigned i;
    3173        1017 :   for (i=0; i < NumElts; ++i)
    3174         756 :     Ops[i] =
    3175         756 :         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    3176             :                     DAG.getConstant(IdxVal + i, dl,
    3177         756 :                                     TLI.getVectorIdxTy(DAG.getDataLayout())));
    3178             : 
    3179         261 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    3180        1225 :   for (; i < WidenNumElts; ++i)
    3181        1928 :     Ops[i] = UndefVal;
    3182         522 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    3183             : }
    3184             : 
    3185          40 : SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
    3186          80 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3187          40 :   return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
    3188             :                      InOp.getValueType(), InOp,
    3189          80 :                      N->getOperand(1), N->getOperand(2));
    3190             : }
    3191             : 
    3192        2865 : SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
    3193             :   LoadSDNode *LD = cast<LoadSDNode>(N);
    3194             :   ISD::LoadExtType ExtType = LD->getExtensionType();
    3195             : 
    3196             :   SDValue Result;
    3197             :   SmallVector<SDValue, 16> LdChain;  // Chain for the series of load
    3198        2865 :   if (ExtType != ISD::NON_EXTLOAD)
    3199           0 :     Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
    3200             :   else
    3201        2865 :     Result = GenWidenVectorLoads(LdChain, LD);
    3202             : 
    3203             :   // If we generate a single load, we can use that for the chain.  Otherwise,
    3204             :   // build a factor node to remember the multiple loads are independent and
    3205             :   // chain to that.
    3206        2865 :   SDValue NewChain;
    3207        5730 :   if (LdChain.size() == 1)
    3208         621 :     NewChain = LdChain[0];
    3209             :   else
    3210        2244 :     NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
    3211             : 
    3212             :   // Modified the chain - switch anything that used the old chain to use
    3213             :   // the new one.
    3214        2865 :   ReplaceValueWith(SDValue(N, 1), NewChain);
    3215             : 
    3216        2865 :   return Result;
    3217             : }
    3218             : 
    3219          10 : SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
    3220             : 
    3221          20 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
    3222          10 :   SDValue Mask = N->getMask();
    3223          10 :   EVT MaskVT = Mask.getValueType();
    3224          10 :   SDValue PassThru = GetWidenedVector(N->getPassThru());
    3225             :   ISD::LoadExtType ExtType = N->getExtensionType();
    3226             :   SDLoc dl(N);
    3227             : 
    3228             :   // The mask should be widened as well
    3229          10 :   EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    3230             :                                     MaskVT.getVectorElementType(),
    3231          10 :                                     WidenVT.getVectorNumElements());
    3232          10 :   Mask = ModifyToType(Mask, WideMaskVT, true);
    3233             : 
    3234          10 :   SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
    3235             :                                   Mask, PassThru, N->getMemoryVT(),
    3236             :                                   N->getMemOperand(), ExtType,
    3237          20 :                                   N->isExpandingLoad());
    3238             :   // Legalize the chain result - switch anything that used the old chain to
    3239             :   // use the new one.
    3240          10 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    3241          10 :   return Res;
    3242             : }
    3243             : 
    3244          18 : SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
    3245             : 
    3246          36 :   EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3247          18 :   SDValue Mask = N->getMask();
    3248          18 :   EVT MaskVT = Mask.getValueType();
    3249          18 :   SDValue PassThru = GetWidenedVector(N->getPassThru());
    3250          18 :   SDValue Scale = N->getScale();
    3251             :   unsigned NumElts = WideVT.getVectorNumElements();
    3252             :   SDLoc dl(N);
    3253             : 
    3254             :   // The mask should be widened as well
    3255          18 :   EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    3256             :                                     MaskVT.getVectorElementType(),
    3257          18 :                                     WideVT.getVectorNumElements());
    3258          18 :   Mask = ModifyToType(Mask, WideMaskVT, true);
    3259             : 
    3260             :   // Widen the Index operand
    3261          18 :   SDValue Index = N->getIndex();
    3262          18 :   EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
    3263          36 :                                      Index.getValueType().getScalarType(),
    3264          18 :                                      NumElts);
    3265          18 :   Index = ModifyToType(Index, WideIndexVT);
    3266             :   SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
    3267          18 :                     Scale };
    3268          18 :   SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
    3269             :                                     N->getMemoryVT(), dl, Ops,
    3270          36 :                                     N->getMemOperand());
    3271             : 
    3272             :   // Legalize the chain result - switch anything that used the old chain to
    3273             :   // use the new one.
    3274          18 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    3275          18 :   return Res;
    3276             : }
    3277             : 
    3278           6 : SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
    3279          12 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3280           6 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
    3281           6 :                      WidenVT, N->getOperand(0));
    3282             : }
    3283             : 
    3284             : // Return true if this is a node that could have two SETCCs as operands.
    3285             : static inline bool isLogicalMaskOp(unsigned Opcode) {
    3286         877 :   switch (Opcode) {
    3287             :   case ISD::AND:
    3288             :   case ISD::OR:
    3289             :   case ISD::XOR:
    3290             :     return true;
    3291             :   }
    3292             :   return false;
    3293             : }
    3294             : 
    3295             : // This is used just for the assert in convertMask(). Check that this either
    3296             : // a SETCC or a previously handled SETCC by convertMask().
    3297             : #ifndef NDEBUG
    3298             : static inline bool isSETCCorConvertedSETCC(SDValue N) {
    3299             :   if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
    3300             :     N = N.getOperand(0);
    3301             :   else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
    3302             :     for (unsigned i = 1; i < N->getNumOperands(); ++i)
    3303             :       if (!N->getOperand(i)->isUndef())
    3304             :         return false;
    3305             :     N = N.getOperand(0);
    3306             :   }
    3307             : 
    3308             :   if (N.getOpcode() == ISD::TRUNCATE)
    3309             :     N = N.getOperand(0);
    3310             :   else if (N.getOpcode() == ISD::SIGN_EXTEND)
    3311             :     N = N.getOperand(0);
    3312             : 
    3313             :   if (isLogicalMaskOp(N.getOpcode()))
    3314             :     return isSETCCorConvertedSETCC(N.getOperand(0)) &&
    3315             :            isSETCCorConvertedSETCC(N.getOperand(1));
    3316             : 
    3317             :   return (N.getOpcode() == ISD::SETCC ||
    3318             :           ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
    3319             : }
    3320             : #endif
    3321             : 
    3322             : // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
    3323             : // to ToMaskVT if needed with vector extension or truncation.
    3324        1121 : SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
    3325             :                                       EVT ToMaskVT) {
    3326             :   // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
    3327             :   // FIXME: This code seems to be too restrictive, we might consider
    3328             :   // generalizing it or dropping it.
    3329             :   assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
    3330             : 
    3331             :   // Make a new Mask node, with a legal result VT.
    3332             :   SmallVector<SDValue, 4> Ops;
    3333        4414 :   for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
    3334        6586 :     Ops.push_back(InMask->getOperand(i));
    3335        1153 :   SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
    3336             : 
    3337             :   // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
    3338             :   // extend or truncate is needed.
    3339        1121 :   LLVMContext &Ctx = *DAG.getContext();
    3340             :   unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
    3341             :   unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
    3342        1121 :   if (MaskScalarBits < ToMaskScalBits) {
    3343             :     EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
    3344          59 :                                  MaskVT.getVectorNumElements());
    3345          59 :     Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
    3346        1062 :   } else if (MaskScalarBits > ToMaskScalBits) {
    3347             :     EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
    3348          87 :                                    MaskVT.getVectorNumElements());
    3349          87 :     Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
    3350             :   }
    3351             : 
    3352             :   assert(Mask->getValueType(0).getScalarSizeInBits() ==
    3353             :              ToMaskVT.getScalarSizeInBits() &&
    3354             :          "Mask should have the right element size by now.");
    3355             : 
    3356             :   // Adjust Mask to the right number of elements.
    3357        3363 :   unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
    3358        1121 :   if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
    3359          28 :     MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
    3360          28 :     SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
    3361          56 :     Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
    3362          56 :                        ZeroIdx);
    3363        1093 :   } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
    3364          39 :     unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
    3365          39 :     EVT SubVT = Mask->getValueType(0);
    3366          39 :     SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
    3367          39 :     SubOps[0] = Mask;
    3368          39 :     Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
    3369             :   }
    3370             : 
    3371             :   assert((Mask->getValueType(0) == ToMaskVT) &&
    3372             :          "A mask of ToMaskVT should have been produced by now.");
    3373             : 
    3374        1121 :   return Mask;
    3375             : }
    3376             : 
    3377             : // Get the target mask VT, and widen if needed.
    3378        1051 : EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
    3379             :   assert(SetCC->getOpcode() == ISD::SETCC);
    3380        1051 :   LLVMContext &Ctx = *DAG.getContext();
    3381        2102 :   EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
    3382        1051 :   if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
    3383          62 :     MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
    3384        1051 :   return MaskVT;
    3385             : }
    3386             : 
    3387             : // This method tries to handle VSELECT and its mask by legalizing operands
    3388             : // (which may require widening) and if needed adjusting the mask vector type
    3389             : // to match that of the VSELECT. Without it, many cases end up with
    3390             : // scalarization of the SETCC, with many unnecessary instructions.
    3391        1826 : SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
    3392        1826 :   LLVMContext &Ctx = *DAG.getContext();
    3393        1826 :   SDValue Cond = N->getOperand(0);
    3394             : 
    3395        1826 :   if (N->getOpcode() != ISD::VSELECT)
    3396           0 :     return SDValue();
    3397             : 
    3398        3652 :   if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
    3399         733 :     return SDValue();
    3400             : 
    3401             :   // If this is a splitted VSELECT that was previously already handled, do
    3402             :   // nothing.
    3403        2186 :   EVT CondVT = Cond->getValueType(0);
    3404        1093 :   if (CondVT.getScalarSizeInBits() != 1)
    3405           1 :     return SDValue();
    3406             : 
    3407        1092 :   EVT VSelVT = N->getValueType(0);
    3408             :   // Only handle vector types which are a power of 2.
    3409        1092 :   if (!isPowerOf2_64(VSelVT.getSizeInBits()))
    3410          28 :     return SDValue();
    3411             : 
    3412             :   // Don't touch if this will be scalarized.
    3413        1064 :   EVT FinalVT = VSelVT;
    3414        1697 :   while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
    3415         633 :     FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
    3416             : 
    3417        1064 :   if (FinalVT.getVectorNumElements() == 1)
    3418          70 :     return SDValue();
    3419             : 
    3420             :   // If there is support for an i1 vector mask, don't touch.
    3421         994 :   if (Cond.getOpcode() == ISD::SETCC) {
    3422         920 :     EVT SetCCOpVT = Cond->getOperand(0).getValueType();
    3423        1492 :     while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
    3424         572 :       SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
    3425         920 :     EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
    3426         920 :     if (SetCCResVT.getScalarSizeInBits() == 1)
    3427           9 :       return SDValue();
    3428         148 :   } else if (CondVT.getScalarType() == MVT::i1) {
    3429             :     // If there is support for an i1 vector mask (or only scalar i1 conditions),
    3430             :     // don't touch.
    3431         152 :     while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
    3432          78 :       CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
    3433             : 
    3434         144 :     if (CondVT.getScalarType() == MVT::i1)
    3435           4 :       return SDValue();
    3436             :   }
    3437             : 
    3438             :   // Get the VT and operands for VSELECT, and widen if needed.
    3439         981 :   SDValue VSelOp1 = N->getOperand(1);
    3440         981 :   SDValue VSelOp2 = N->getOperand(2);
    3441         981 :   if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
    3442          61 :     VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
    3443          61 :     VSelOp1 = GetWidenedVector(VSelOp1);
    3444          61 :     VSelOp2 = GetWidenedVector(VSelOp2);
    3445             :   }
    3446             : 
    3447             :   // The mask of the VSELECT should have integer elements.
    3448         981 :   EVT ToMaskVT = VSelVT;
    3449         981 :   if (!ToMaskVT.getScalarType().isInteger())
    3450         452 :     ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
    3451             : 
    3452         981 :   SDValue Mask;
    3453        1962 :   if (Cond->getOpcode() == ISD::SETCC) {
    3454         911 :     EVT MaskVT = getSETCCWidenedResultTy(Cond);
    3455         911 :     Mask = convertMask(Cond, MaskVT, ToMaskVT);
    3456          70 :   } else if (isLogicalMaskOp(Cond->getOpcode()) &&
    3457          70 :              Cond->getOperand(0).getOpcode() == ISD::SETCC &&
    3458          70 :              Cond->getOperand(1).getOpcode() == ISD::SETCC) {
    3459             :     // Cond is (AND/OR/XOR (SETCC, SETCC))
    3460          70 :     SDValue SETCC0 = Cond->getOperand(0);
    3461          70 :     SDValue SETCC1 = Cond->getOperand(1);
    3462          70 :     EVT VT0 = getSETCCWidenedResultTy(SETCC0);
    3463          70 :     EVT VT1 = getSETCCWidenedResultTy(SETCC1);
    3464             :     unsigned ScalarBits0 = VT0.getScalarSizeInBits();
    3465             :     unsigned ScalarBits1 = VT1.getScalarSizeInBits();
    3466             :     unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
    3467             :     EVT MaskVT;
    3468             :     // If the two SETCCs have different VTs, either extend/truncate one of
    3469             :     // them to the other "towards" ToMaskVT, or truncate one and extend the
    3470             :     // other to ToMaskVT.
    3471          70 :     if (ScalarBits0 != ScalarBits1) {
    3472          40 :       EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
    3473          80 :       EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
    3474          40 :       if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
    3475          18 :         MaskVT = WideVT;
    3476          22 :       else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
    3477          22 :         MaskVT = NarrowVT;
    3478             :       else
    3479           0 :         MaskVT = ToMaskVT;
    3480             :     } else
    3481             :       // If the two SETCCs have the same VT, don't change it.
    3482          30 :       MaskVT = VT0;
    3483             : 
    3484             :     // Make new SETCCs and logical nodes.
    3485          70 :     SETCC0 = convertMask(SETCC0, VT0, MaskVT);
    3486          70 :     SETCC1 = convertMask(SETCC1, VT1, MaskVT);
    3487          70 :     Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
    3488             : 
    3489             :     // Convert the logical op for VSELECT if needed.
    3490          70 :     Mask = convertMask(Cond, MaskVT, ToMaskVT);
    3491             :   } else
    3492           0 :     return SDValue();
    3493             : 
    3494        1978 :   return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
    3495             : }
    3496             : 
    3497         119 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
    3498         238 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3499             :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3500             : 
    3501         119 :   SDValue Cond1 = N->getOperand(0);
    3502         119 :   EVT CondVT = Cond1.getValueType();
    3503         119 :   if (CondVT.isVector()) {
    3504         107 :     if (SDValue Res = WidenVSELECTAndMask(N))
    3505          61 :       return Res;
    3506             : 
    3507          46 :     EVT CondEltVT = CondVT.getVectorElementType();
    3508          46 :     EVT CondWidenVT =  EVT::getVectorVT(*DAG.getContext(),
    3509          46 :                                         CondEltVT, WidenNumElts);
    3510          46 :     if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
    3511          34 :       Cond1 = GetWidenedVector(Cond1);
    3512             : 
    3513             :     // If we have to split the condition there is no point in widening the
    3514             :     // select. This would result in an cycle of widening the select ->
    3515             :     // widening the condition operand -> splitting the condition operand ->
    3516             :     // splitting the select -> widening the select. Instead split this select
    3517             :     // further and widen the resulting type.
    3518          46 :     if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
    3519           4 :       SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
    3520           4 :       SDValue Res = ModifyToType(SplitSelect, WidenVT);
    3521           4 :       return Res;
    3522             :     }
    3523             : 
    3524          42 :     if (Cond1.getValueType() != CondWidenVT)
    3525           8 :       Cond1 = ModifyToType(Cond1, CondWidenVT);
    3526             :   }
    3527             : 
    3528         108 :   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
    3529         108 :   SDValue InOp2 = GetWidenedVector(N->getOperand(2));
    3530             :   assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
    3531          54 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
    3532         162 :                      WidenVT, Cond1, InOp1, InOp2);
    3533             : }
    3534             : 
    3535           0 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
    3536           0 :   SDValue InOp1 = GetWidenedVector(N->getOperand(2));
    3537           0 :   SDValue InOp2 = GetWidenedVector(N->getOperand(3));
    3538           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
    3539             :                      InOp1.getValueType(), N->getOperand(0),
    3540           0 :                      N->getOperand(1), InOp1, InOp2, N->getOperand(4));
    3541             : }
    3542             : 
    3543         760 : SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
    3544        1520 :  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3545         760 :  return DAG.getUNDEF(WidenVT);
    3546             : }
    3547             : 
    3548         116 : SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
    3549         232 :   EVT VT = N->getValueType(0);
    3550             :   SDLoc dl(N);
    3551             : 
    3552         116 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3553             :   unsigned NumElts = VT.getVectorNumElements();
    3554             :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3555             : 
    3556         232 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    3557         232 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    3558             : 
    3559             :   // Adjust mask based on new input vector length.
    3560             :   SmallVector<int, 16> NewMask;
    3561         479 :   for (unsigned i = 0; i != NumElts; ++i) {
    3562         363 :     int Idx = N->getMaskElt(i);
    3563         363 :     if (Idx < (int)NumElts)
    3564         302 :       NewMask.push_back(Idx);
    3565             :     else
    3566          61 :       NewMask.push_back(Idx - NumElts + WidenNumElts);
    3567             :   }
    3568         517 :   for (unsigned i = NumElts; i != WidenNumElts; ++i)
    3569         401 :     NewMask.push_back(-1);
    3570         232 :   return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
    3571             : }
    3572             : 
    3573         114 : SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
    3574             :   assert(N->getValueType(0).isVector() &&
    3575             :          N->getOperand(0).getValueType().isVector() &&
    3576             :          "Operands must be vectors");
    3577         228 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3578             :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3579             : 
    3580         114 :   SDValue InOp1 = N->getOperand(0);
    3581         114 :   EVT InVT = InOp1.getValueType();
    3582             :   assert(InVT.isVector() && "can not widen non-vector type");
    3583         114 :   EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
    3584         114 :                                    InVT.getVectorElementType(), WidenNumElts);
    3585             : 
    3586             :   // The input and output types often differ here, and it could be that while
    3587             :   // we'd prefer to widen the result type, the input operands have been split.
    3588             :   // In this case, we also need to split the result of this node as well.
    3589         114 :   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
    3590           2 :     SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
    3591           2 :     SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
    3592           2 :     return Res;
    3593             :   }
    3594             : 
    3595         112 :   InOp1 = GetWidenedVector(InOp1);
    3596         224 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    3597             : 
    3598             :   // Assume that the input and output will be widen appropriately.  If not,
    3599             :   // we will have to unroll it at some point.
    3600             :   assert(InOp1.getValueType() == WidenInVT &&
    3601             :          InOp2.getValueType() == WidenInVT &&
    3602             :          "Input not widened to expected type!");
    3603             :   (void)WidenInVT;
    3604         112 :   return DAG.getNode(ISD::SETCC, SDLoc(N),
    3605         224 :                      WidenVT, InOp1, InOp2, N->getOperand(2));
    3606             : }
    3607             : 
    3608             : 
    3609             : //===----------------------------------------------------------------------===//
    3610             : // Widen Vector Operand
    3611             : //===----------------------------------------------------------------------===//
    3612       11892 : bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
    3613             :   LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
    3614             :              dbgs() << "\n");
    3615       11892 :   SDValue Res = SDValue();
    3616             : 
    3617             :   // See if the target wants to custom widen this node.
    3618       35676 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    3619             :     return false;
    3620             : 
    3621       23542 :   switch (N->getOpcode()) {
    3622           0 :   default:
    3623             : #ifndef NDEBUG
    3624             :     dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
    3625             :     N->dump(&DAG);
    3626             :     dbgs() << "\n";
    3627             : #endif
    3628           0 :     llvm_unreachable("Do not know how to widen this operator's operand!");
    3629             : 
    3630          12 :   case ISD::BITCAST:            Res = WidenVecOp_BITCAST(N); break;
    3631        1199 :   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
    3632          72 :   case ISD::EXTRACT_SUBVECTOR:  Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
    3633        9123 :   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
    3634         920 :   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
    3635          10 :   case ISD::MSTORE:             Res = WidenVecOp_MSTORE(N, OpNo); break;
    3636           3 :   case ISD::MGATHER:            Res = WidenVecOp_MGATHER(N, OpNo); break;
    3637           5 :   case ISD::MSCATTER:           Res = WidenVecOp_MSCATTER(N, OpNo); break;
    3638          74 :   case ISD::SETCC:              Res = WidenVecOp_SETCC(N); break;
    3639           3 :   case ISD::FCOPYSIGN:          Res = WidenVecOp_FCOPYSIGN(N); break;
    3640             : 
    3641         235 :   case ISD::ANY_EXTEND:
    3642             :   case ISD::SIGN_EXTEND:
    3643             :   case ISD::ZERO_EXTEND:
    3644         235 :     Res = WidenVecOp_EXTEND(N);
    3645         235 :     break;
    3646             : 
    3647         115 :   case ISD::FP_EXTEND:
    3648             :   case ISD::FP_TO_SINT:
    3649             :   case ISD::FP_TO_UINT:
    3650             :   case ISD::SINT_TO_FP:
    3651             :   case ISD::UINT_TO_FP:
    3652             :   case ISD::TRUNCATE:
    3653         115 :     Res = WidenVecOp_Convert(N);
    3654         115 :     break;
    3655             :   }
    3656             : 
    3657             :   // If Res is null, the sub-method took care of registering the result.
    3658       11771 :   if (!Res.getNode()) return false;
    3659             : 
    3660             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    3661             :   // core about this.
    3662       11768 :   if (Res.getNode() == N)
    3663             :     return true;
    3664             : 
    3665             : 
    3666             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    3667             :          "Invalid operand expansion");
    3668             : 
    3669       11768 :   ReplaceValueWith(SDValue(N, 0), Res);
    3670       11768 :   return false;
    3671             : }
    3672             : 
    3673         235 : SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
    3674             :   SDLoc DL(N);
    3675         235 :   EVT VT = N->getValueType(0);
    3676             : 
    3677         235 :   SDValue InOp = N->getOperand(0);
    3678             :   assert(getTypeAction(InOp.getValueType()) ==
    3679             :              TargetLowering::TypeWidenVector &&
    3680             :          "Unexpected type action");
    3681         235 :   InOp = GetWidenedVector(InOp);
    3682             :   assert(VT.getVectorNumElements() <
    3683             :              InOp.getValueType().getVectorNumElements() &&
    3684             :          "Input wasn't widened!");
    3685             : 
    3686             :   // We may need to further widen the operand until it has the same total
    3687             :   // vector size as the result.
    3688         235 :   EVT InVT = InOp.getValueType();
    3689         235 :   if (InVT.getSizeInBits() != VT.getSizeInBits()) {
    3690           0 :     EVT InEltVT = InVT.getVectorElementType();
    3691           0 :     for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
    3692           0 :       EVT FixedVT = (MVT::SimpleValueType)i;
    3693           0 :       EVT FixedEltVT = FixedVT.getVectorElementType();
    3694           0 :       if (TLI.isTypeLegal(FixedVT) &&
    3695           0 :           FixedVT.getSizeInBits() == VT.getSizeInBits() &&
    3696           0 :           FixedEltVT == InEltVT) {
    3697             :         assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
    3698             :                "Not enough elements in the fixed type for the operand!");
    3699             :         assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
    3700             :                "We can't have the same type as we started with!");
    3701           0 :         if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
    3702           0 :           InOp = DAG.getNode(
    3703           0 :               ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
    3704           0 :               DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3705             :         else
    3706           0 :           InOp = DAG.getNode(
    3707             :               ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
    3708           0 :               DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3709           0 :         break;
    3710             :       }
    3711             :     }
    3712           0 :     InVT = InOp.getValueType();
    3713           0 :     if (InVT.getSizeInBits() != VT.getSizeInBits())
    3714             :       // We couldn't find a legal vector type that was a widening of the input
    3715             :       // and could be extended in-register to the result type, so we have to
    3716             :       // scalarize.
    3717           0 :       return WidenVecOp_Convert(N);
    3718             :   }
    3719             : 
    3720             :   // Use special DAG nodes to represent the operation of extending the
    3721             :   // low lanes.
    3722         470 :   switch (N->getOpcode()) {
    3723           0 :   default:
    3724           0 :     llvm_unreachable("Extend legalization on extend operation!");
    3725           2 :   case ISD::ANY_EXTEND:
    3726           2 :     return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
    3727         197 :   case ISD::SIGN_EXTEND:
    3728         197 :     return DAG.getSignExtendVectorInReg(InOp, DL, VT);
    3729          36 :   case ISD::ZERO_EXTEND:
    3730          36 :     return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
    3731             :   }
    3732             : }
    3733             : 
    3734           0 : SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
    3735             :   // The result (and first input) is legal, but the second input is illegal.
    3736             :   // We can't do much to fix that, so just unroll and let the extracts off of
    3737             :   // the second input be widened as needed later.
    3738           3 :   return DAG.UnrollVectorOp(N);
    3739             : }
    3740             : 
    3741         115 : SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
    3742             :   // Since the result is legal and the input is illegal.
    3743         115 :   EVT VT = N->getValueType(0);
    3744         115 :   EVT EltVT = VT.getVectorElementType();
    3745             :   SDLoc dl(N);
    3746             :   unsigned NumElts = VT.getVectorNumElements();
    3747         115 :   SDValue InOp = N->getOperand(0);
    3748             :   assert(getTypeAction(InOp.getValueType()) ==
    3749             :              TargetLowering::TypeWidenVector &&
    3750             :          "Unexpected type action");
    3751         115 :   InOp = GetWidenedVector(InOp);
    3752         115 :   EVT InVT = InOp.getValueType();
    3753         115 :   unsigned Opcode = N->getOpcode();
    3754             : 
    3755             :   // See if a widened result type would be legal, if so widen the node.
    3756         115 :   EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
    3757         115 :                                 InVT.getVectorNumElements());
    3758         115 :   if (TLI.isTypeLegal(WideVT)) {
    3759          70 :     SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp);
    3760          35 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
    3761          35 :                        DAG.getIntPtrConstant(0, dl));
    3762             :   }
    3763             : 
    3764          80 :   EVT InEltVT = InVT.getVectorElementType();
    3765             : 
    3766             :   // Unroll the convert into some scalar code and create a nasty build vector.
    3767          80 :   SmallVector<SDValue, 16> Ops(NumElts);
    3768         244 :   for (unsigned i=0; i < NumElts; ++i)
    3769         164 :     Ops[i] = DAG.getNode(
    3770             :         Opcode, dl, EltVT,
    3771             :         DAG.getNode(
    3772             :             ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
    3773         164 :             DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3774             : 
    3775         160 :   return DAG.getBuildVector(VT, dl, Ops);
    3776             : }
    3777             : 
    3778          12 : SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
    3779          12 :   EVT VT = N->getValueType(0);
    3780          24 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3781          24 :   EVT InWidenVT = InOp.getValueType();
    3782             :   SDLoc dl(N);
    3783             : 
    3784             :   // Check if we can convert between two legal vector types and extract.
    3785          12 :   unsigned InWidenSize = InWidenVT.getSizeInBits();
    3786          12 :   unsigned Size = VT.getSizeInBits();
    3787             :   // x86mmx is not an acceptable vector element type, so don't try.
    3788          24 :   if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
    3789          12 :     unsigned NewNumElts = InWidenSize / Size;
    3790          12 :     EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
    3791          12 :     if (TLI.isTypeLegal(NewVT)) {
    3792          24 :       SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
    3793          12 :       return DAG.getNode(
    3794             :           ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
    3795          12 :           DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3796             :     }
    3797             :   }
    3798             : 
    3799           0 :   return CreateStackStoreLoad(InOp, VT);
    3800             : }
    3801             : 
    3802        1199 : SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
    3803             :   // If the input vector is not legal, it is likely that we will not find a
    3804             :   // legal vector of the same size. Replace the concatenate vector with a
    3805             :   // nasty build vector.
    3806        1199 :   EVT VT = N->getValueType(0);
    3807        1199 :   EVT EltVT = VT.getVectorElementType();
    3808             :   SDLoc dl(N);
    3809             :   unsigned NumElts = VT.getVectorNumElements();
    3810        1199 :   SmallVector<SDValue, 16> Ops(NumElts);
    3811             : 
    3812        2398 :   EVT InVT = N->getOperand(0).getValueType();
    3813             :   unsigned NumInElts = InVT.getVectorNumElements();
    3814             : 
    3815             :   unsigned Idx = 0;
    3816        1199 :   unsigned NumOperands = N->getNumOperands();
    3817        3955 :   for (unsigned i=0; i < NumOperands; ++i) {
    3818        2756 :     SDValue InOp = N->getOperand(i);
    3819             :     assert(getTypeAction(InOp.getValueType()) ==
    3820             :                TargetLowering::TypeWidenVector &&
    3821             :            "Unexpected type action");
    3822        2756 :     InOp = GetWidenedVector(InOp);
    3823        9134 :     for (unsigned j=0; j < NumInElts; ++j)
    3824       12756 :       Ops[Idx++] = DAG.getNode(
    3825             :           ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    3826        6378 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3827             :   }
    3828        2398 :   return DAG.getBuildVector(VT, dl, Ops);
    3829             : }
    3830             : 
    3831          72 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
    3832         144 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3833          72 :   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
    3834         144 :                      N->getValueType(0), InOp, N->getOperand(1));
    3835             : }
    3836             : 
    3837        9123 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    3838       18246 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3839        9123 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
    3840       18249 :                      N->getValueType(0), InOp, N->getOperand(1));
    3841             : }
    3842             : 
    3843         920 : SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
    3844             :   // We have to widen the value, but we want only to store the original
    3845             :   // vector type.
    3846             :   StoreSDNode *ST = cast<StoreSDNode>(N);
    3847             : 
    3848         920 :   if (!ST->getMemoryVT().getScalarType().isByteSized())
    3849          12 :     return TLI.scalarizeVectorStore(ST, DAG);
    3850             : 
    3851             :   SmallVector<SDValue, 16> StChain;
    3852         908 :   if (ST->isTruncatingStore())
    3853           0 :     GenWidenVectorTruncStores(StChain, ST);
    3854             :   else
    3855         908 :     GenWidenVectorStores(StChain, ST);
    3856             : 
    3857        1816 :   if (StChain.size() == 1)
    3858         230 :     return StChain[0];
    3859             :   else
    3860        1356 :     return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
    3861             : }
    3862             : 
    3863          10 : SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
    3864             :   assert((OpNo == 1 || OpNo == 3) &&
    3865             :          "Can widen only data or mask operand of mstore");
    3866             :   MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
    3867          10 :   SDValue Mask = MST->getMask();
    3868          10 :   EVT MaskVT = Mask.getValueType();
    3869          10 :   SDValue StVal = MST->getValue();
    3870             :   SDLoc dl(N);
    3871             : 
    3872          10 :   if (OpNo == 1) {
    3873             :     // Widen the value.
    3874          10 :     StVal = GetWidenedVector(StVal);
    3875             : 
    3876             :     // The mask should be widened as well.
    3877          10 :     EVT WideVT = StVal.getValueType();
    3878          10 :     EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    3879             :                                       MaskVT.getVectorElementType(),
    3880          10 :                                       WideVT.getVectorNumElements());
    3881          10 :     Mask = ModifyToType(Mask, WideMaskVT, true);
    3882             :   } else {
    3883             :     // Widen the mask.
    3884           0 :     EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
    3885           0 :     Mask = ModifyToType(Mask, WideMaskVT, true);
    3886             : 
    3887           0 :     EVT ValueVT = StVal.getValueType();
    3888           0 :     EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
    3889             :                                   ValueVT.getVectorElementType(),
    3890           0 :                                   WideMaskVT.getVectorNumElements());
    3891           0 :     StVal = ModifyToType(StVal, WideVT);
    3892             :   }
    3893             : 
    3894             :   assert(Mask.getValueType().getVectorNumElements() ==
    3895             :          StVal.getValueType().getVectorNumElements() &&
    3896             :          "Mask and data vectors should have the same number of elements");
    3897          10 :   return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
    3898             :                             Mask, MST->getMemoryVT(), MST->getMemOperand(),
    3899          20 :                             false, MST->isCompressingStore());
    3900             : }
    3901             : 
    3902           3 : SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
    3903             :   assert(OpNo == 4 && "Can widen only the index of mgather");
    3904             :   auto *MG = cast<MaskedGatherSDNode>(N);
    3905           3 :   SDValue DataOp = MG->getPassThru();
    3906           3 :   SDValue Mask = MG->getMask();
    3907           3 :   SDValue Scale = MG->getScale();
    3908             : 
    3909             :   // Just widen the index. It's allowed to have extra elements.
    3910           3 :   SDValue Index = GetWidenedVector(MG->getIndex());
    3911             : 
    3912             :   SDLoc dl(N);
    3913             :   SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
    3914           3 :                    Scale};
    3915           3 :   SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
    3916           9 :                                     MG->getMemOperand());
    3917           3 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    3918           3 :   ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
    3919           3 :   return SDValue();
    3920             : }
    3921             : 
    3922           5 : SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
    3923             :   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
    3924           5 :   SDValue DataOp = MSC->getValue();
    3925           5 :   SDValue Mask = MSC->getMask();
    3926           5 :   SDValue Index = MSC->getIndex();
    3927           5 :   SDValue Scale = MSC->getScale();
    3928             : 
    3929             :   unsigned NumElts;
    3930           5 :   if (OpNo == 1) {
    3931           3 :     DataOp = GetWidenedVector(DataOp);
    3932           6 :     NumElts = DataOp.getValueType().getVectorNumElements();
    3933             : 
    3934             :     // Widen index.
    3935           3 :     EVT IndexVT = Index.getValueType();
    3936           3 :     EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
    3937           3 :                                        IndexVT.getVectorElementType(), NumElts);
    3938           3 :     Index = ModifyToType(Index, WideIndexVT);
    3939             : 
    3940             :     // The mask should be widened as well.
    3941           3 :     EVT MaskVT = Mask.getValueType();
    3942           3 :     EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    3943           3 :                                       MaskVT.getVectorElementType(), NumElts);
    3944           3 :     Mask = ModifyToType(Mask, WideMaskVT, true);
    3945           2 :   } else if (OpNo == 4) {
    3946             :     // Just widen the index. It's allowed to have extra elements.
    3947           2 :     Index = GetWidenedVector(Index);
    3948             :   } else
    3949           0 :     llvm_unreachable("Can't widen this operand of mscatter");
    3950             : 
    3951             :   SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
    3952           5 :                    Scale};
    3953           5 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
    3954           5 :                               MSC->getMemoryVT(), SDLoc(N), Ops,
    3955           5 :                               MSC->getMemOperand());
    3956             : }
    3957             : 
    3958          74 : SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
    3959         148 :   SDValue InOp0 = GetWidenedVector(N->getOperand(0));
    3960         148 :   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
    3961             :   SDLoc dl(N);
    3962          74 :   EVT VT = N->getValueType(0);
    3963             : 
    3964             :   // WARNING: In this code we widen the compare instruction with garbage.
    3965             :   // This garbage may contain denormal floats which may be slow. Is this a real
    3966             :   // concern ? Should we zero the unused lanes if this is a float compare ?
    3967             : 
    3968             :   // Get a new SETCC node to compare the newly widened operands.
    3969             :   // Only some of the compared elements are legal.
    3970         148 :   EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
    3971         148 :                                    InOp0.getValueType());
    3972             :   // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
    3973         148 :   if (VT.getScalarType() == MVT::i1)
    3974           3 :     SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
    3975           3 :                            SVT.getVectorNumElements());
    3976             : 
    3977          74 :   SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
    3978         148 :                                   SVT, InOp0, InOp1, N->getOperand(2));
    3979             : 
    3980             :   // Extract the needed results from the result vector.
    3981          74 :   EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
    3982             :                                SVT.getVectorElementType(),
    3983          74 :                                VT.getVectorNumElements());
    3984          74 :   SDValue CC = DAG.getNode(
    3985             :       ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
    3986          74 :       DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3987             : 
    3988          74 :   return PromoteTargetBoolean(CC, VT);
    3989             : }
    3990             : 
    3991             : 
    3992             : //===----------------------------------------------------------------------===//
    3993             : // Vector Widening Utilities
    3994             : //===----------------------------------------------------------------------===//
    3995             : 
    3996             : // Utility function to find the type to chop up a widen vector for load/store
    3997             : //  TLI:       Target lowering used to determine legal types.
    3998             : //  Width:     Width left need to load/store.
    3999             : //  WidenVT:   The widen vector type to load to/store from
    4000             : //  Align:     If 0, don't allow use of a wider type
    4001             : //  WidenEx:   If Align is not 0, the amount additional we can load/store from.
    4002             : 
    4003        6522 : static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
    4004             :                        unsigned Width, EVT WidenVT,
    4005             :                        unsigned Align = 0, unsigned WidenEx = 0) {
    4006        6522 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    4007        6522 :   unsigned WidenWidth = WidenVT.getSizeInBits();
    4008        6522 :   unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
    4009        6522 :   unsigned AlignInBits = Align*8;
    4010             : 
    4011             :   // If we have one element to load/store, return it.
    4012        6522 :   EVT RetVT = WidenEltVT;
    4013        6522 :   if (Width == WidenEltWidth)
    4014        1166 :     return RetVT;
    4015             : 
    4016             :   // See if there is larger legal integer than the element type to load/store.
    4017             :   unsigned VT;
    4018        6103 :   for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
    4019       11459 :        VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
    4020       11459 :     EVT MemVT((MVT::SimpleValueType) VT);
    4021       11459 :     unsigned MemVTWidth = MemVT.getSizeInBits();
    4022       11459 :     if (MemVT.getSizeInBits() <= WidenEltWidth)
    4023             :       break;
    4024       11088 :     auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
    4025             :     if ((Action == TargetLowering::TypeLegal ||
    4026        5423 :          Action == TargetLowering::TypePromoteInteger) &&
    4027        5423 :         (WidenWidth % MemVTWidth) == 0 &&
    4028       21464 :         isPowerOf2_32(WidenWidth / MemVTWidth) &&
    4029         348 :         (MemVTWidth <= Width ||
    4030         348 :          (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
    4031        4985 :       RetVT = MemVT;
    4032        4985 :       break;
    4033             :     }
    4034             :   }
    4035             : 
    4036             :   // See if there is a larger vector type to load/store that has the same vector
    4037             :   // element type and is evenly divisible with the WidenVT.
    4038      444741 :   for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
    4039      450097 :        VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
    4040      446842 :     EVT MemVT = (MVT::SimpleValueType) VT;
    4041      446842 :     unsigned MemVTWidth = MemVT.getSizeInBits();
    4042       55660 :     if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
    4043       15149 :         (WidenWidth % MemVTWidth) == 0 &&
    4044       20622 :         isPowerOf2_32(WidenWidth / MemVTWidth) &&
    4045        6319 :         (MemVTWidth <= Width ||
    4046        6319 :          (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
    4047        4286 :       if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
    4048        2101 :         return MemVT;
    4049             :     }
    4050             :   }
    4051             : 
    4052        3255 :   return RetVT;
    4053             : }
    4054             : 
    4055             : // Builds a vector type from scalar loads
    4056             : //  VecTy: Resulting Vector type
    4057             : //  LDOps: Load operators to build a vector type
    4058             : //  [Start,End) the list of loads to use.
    4059        2106 : static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
    4060             :                                      SmallVectorImpl<SDValue> &LdOps,
    4061             :                                      unsigned Start, unsigned End) {
    4062        2106 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    4063        2106 :   SDLoc dl(LdOps[Start]);
    4064        2106 :   EVT LdTy = LdOps[Start].getValueType();
    4065        2106 :   unsigned Width = VecTy.getSizeInBits();
    4066        2106 :   unsigned NumElts = Width / LdTy.getSizeInBits();
    4067        2106 :   EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
    4068             : 
    4069             :   unsigned Idx = 1;
    4070        2106 :   SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
    4071             : 
    4072        2963 :   for (unsigned i = Start + 1; i != End; ++i) {
    4073        1714 :     EVT NewLdTy = LdOps[i].getValueType();
    4074         883 :     if (NewLdTy != LdTy) {
    4075         664 :       NumElts = Width / NewLdTy.getSizeInBits();
    4076         664 :       NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
    4077         664 :       VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
    4078             :       // Readjust position and vector position based on new load type.
    4079         664 :       Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
    4080         664 :       LdTy = NewLdTy;
    4081             :     }
    4082         857 :     VecOp = DAG.getNode(
    4083             :         ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
    4084         857 :         DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4085             :   }
    4086        2106 :   return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
    4087             : }
    4088             : 
    4089        2865 : SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
    4090             :                                               LoadSDNode *LD) {
    4091             :   // The strategy assumes that we can efficiently load power-of-two widths.
    4092             :   // The routine chops the vector into the largest vector loads with the same
    4093             :   // element type or scalar loads and then recombines it to the widen vector
    4094             :   // type.
    4095        5730 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
    4096        2865 :   unsigned WidenWidth = WidenVT.getSizeInBits();
    4097        2865 :   EVT LdVT    = LD->getMemoryVT();
    4098             :   SDLoc dl(LD);
    4099             :   assert(LdVT.isVector() && WidenVT.isVector());
    4100             :   assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
    4101             : 
    4102             :   // Load information
    4103        2865 :   SDValue Chain = LD->getChain();
    4104        2865 :   SDValue BasePtr = LD->getBasePtr();
    4105        2865 :   unsigned Align = LD->getAlignment();
    4106        2865 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    4107             :   AAMDNodes AAInfo = LD->getAAInfo();
    4108             : 
    4109        2865 :   int LdWidth = LdVT.getSizeInBits();
    4110        2865 :   int WidthDiff = WidenWidth - LdWidth;
    4111        2865 :   unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
    4112             : 
    4113             :   // Find the vector type that can load from.
    4114        2865 :   EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
    4115        2865 :   int NewVTWidth = NewVT.getSizeInBits();
    4116        5730 :   SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
    4117        5730 :                              Align, MMOFlags, AAInfo);
    4118        2865 :   LdChain.push_back(LdOp.getValue(1));
    4119             : 
    4120             :   // Check if we can load the element with one instruction.
    4121        2865 :   if (LdWidth <= NewVTWidth) {
    4122         621 :     if (!NewVT.isVector()) {
    4123         438 :       unsigned NumElts = WidenWidth / NewVTWidth;
    4124         438 :       EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
    4125         876 :       SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
    4126         876 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
    4127             :     }
    4128         183 :     if (NewVT == WidenVT)
    4129         183 :       return LdOp;
    4130             : 
    4131             :     assert(WidenWidth % NewVTWidth == 0);
    4132           0 :     unsigned NumConcat = WidenWidth / NewVTWidth;
    4133           0 :     SmallVector<SDValue, 16> ConcatOps(NumConcat);
    4134           0 :     SDValue UndefVal = DAG.getUNDEF(NewVT);
    4135           0 :     ConcatOps[0] = LdOp;
    4136           0 :     for (unsigned i = 1; i != NumConcat; ++i)
    4137           0 :       ConcatOps[i] = UndefVal;
    4138           0 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
    4139             :   }
    4140             : 
    4141             :   // Load vector by using multiple loads from largest vector to scalar.
    4142             :   SmallVector<SDValue, 16> LdOps;
    4143        2244 :   LdOps.push_back(LdOp);
    4144             : 
    4145        2244 :   LdWidth -= NewVTWidth;
    4146             :   unsigned Offset = 0;
    4147             : 
    4148        5002 :   while (LdWidth > 0) {
    4149        2758 :     unsigned Increment = NewVTWidth / 8;
    4150        2758 :     Offset += Increment;
    4151        2758 :     BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
    4152             : 
    4153        2758 :     SDValue L;
    4154        2758 :     if (LdWidth < NewVTWidth) {
    4155             :       // The current type we are using is too large. Find a better size.
    4156        2137 :       NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
    4157        2137 :       NewVTWidth = NewVT.getSizeInBits();
    4158        4274 :       L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
    4159        2137 :                       LD->getPointerInfo().getWithOffset(Offset),
    4160        6411 :                       MinAlign(Align, Increment), MMOFlags, AAInfo);
    4161        2137 :       LdChain.push_back(L.getValue(1));
    4162        6411 :       if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
    4163             :         // Later code assumes the vector loads produced will be mergeable, so we
    4164             :         // must pad the final entry up to the previous width. Scalars are
    4165             :         // combined separately.
    4166             :         SmallVector<SDValue, 16> Loads;
    4167         119 :         Loads.push_back(L);
    4168         119 :         unsigned size = L->getValueSizeInBits(0);
    4169         256 :         while (size < LdOp->getValueSizeInBits(0)) {
    4170          18 :           Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
    4171          18 :           size += L->getValueSizeInBits(0);
    4172             :         }
    4173         238 :         L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
    4174             :       }
    4175             :     } else {
    4176        1242 :       L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
    4177         621 :                       LD->getPointerInfo().getWithOffset(Offset),
    4178        1863 :                       MinAlign(Align, Increment), MMOFlags, AAInfo);
    4179         621 :       LdChain.push_back(L.getValue(1));
    4180             :     }
    4181             : 
    4182        2758 :     LdOps.push_back(L);
    4183        2758 :     LdOp = L;
    4184             : 
    4185        2758 :     LdWidth -= NewVTWidth;
    4186             :   }
    4187             : 
    4188             :   // Build the vector from the load operations.
    4189        2244 :   unsigned End = LdOps.size();
    4190        6732 :   if (!LdOps[0].getValueType().isVector())
    4191             :     // All the loads are scalar loads.
    4192         763 :     return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
    4193             : 
    4194             :   // If the load contains vectors, build the vector using concat vector.
    4195             :   // All of the vectors used to load are power-of-2, and the scalar loads can be
    4196             :   // combined to make a power-of-2 vector.
    4197        1481 :   SmallVector<SDValue, 16> ConcatOps(End);
    4198        1481 :   int i = End - 1;
    4199        1481 :   int Idx = End;
    4200        4443 :   EVT LdTy = LdOps[i].getValueType();
    4201             :   // First, combine the scalar loads to a vector.
    4202        1481 :   if (!LdTy.isVector())  {
    4203        1343 :     for (--i; i >= 0; --i) {
    4204        4029 :       LdTy = LdOps[i].getValueType();
    4205        1343 :       if (LdTy.isVector())
    4206             :         break;
    4207             :     }
    4208        1343 :     ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
    4209             :   }
    4210        2962 :   ConcatOps[--Idx] = LdOps[i];
    4211        2039 :   for (--i; i >= 0; --i) {
    4212        1116 :     EVT NewLdTy = LdOps[i].getValueType();
    4213         558 :     if (NewLdTy != LdTy) {
    4214             :       // Create a larger vector.
    4215          11 :       ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
    4216          22 :                                      makeArrayRef(&ConcatOps[Idx], End - Idx));
    4217             :       Idx = End - 1;
    4218          11 :       LdTy = NewLdTy;
    4219             :     }
    4220        1116 :     ConcatOps[--Idx] = LdOps[i];
    4221             :   }
    4222             : 
    4223        1481 :   if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
    4224        1447 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
    4225        2894 :                        makeArrayRef(&ConcatOps[Idx], End - Idx));
    4226             : 
    4227             :   // We need to fill the rest with undefs to build the vector.
    4228          34 :   unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
    4229          34 :   SmallVector<SDValue, 16> WidenOps(NumOps);
    4230          34 :   SDValue UndefVal = DAG.getUNDEF(LdTy);
    4231             :   {
    4232             :     unsigned i = 0;
    4233         503 :     for (; i != End-Idx; ++i)
    4234        1407 :       WidenOps[i] = ConcatOps[Idx+i];
    4235         229 :     for (; i != NumOps; ++i)
    4236         390 :       WidenOps[i] = UndefVal;
    4237             :   }
    4238          68 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
    4239             : }
    4240             : 
    4241             : SDValue
    4242           0 : DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
    4243             :                                          LoadSDNode *LD,
    4244             :                                          ISD::LoadExtType ExtType) {
    4245             :   // For extension loads, it may not be more efficient to chop up the vector
    4246             :   // and then extend it. Instead, we unroll the load and build a new vector.
    4247           0 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
    4248           0 :   EVT LdVT    = LD->getMemoryVT();
    4249             :   SDLoc dl(LD);
    4250             :   assert(LdVT.isVector() && WidenVT.isVector());
    4251             : 
    4252             :   // Load information
    4253           0 :   SDValue Chain = LD->getChain();
    4254           0 :   SDValue BasePtr = LD->getBasePtr();
    4255           0 :   unsigned Align = LD->getAlignment();
    4256           0 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    4257             :   AAMDNodes AAInfo = LD->getAAInfo();
    4258             : 
    4259           0 :   EVT EltVT = WidenVT.getVectorElementType();
    4260           0 :   EVT LdEltVT = LdVT.getVectorElementType();
    4261             :   unsigned NumElts = LdVT.getVectorNumElements();
    4262             : 
    4263             :   // Load each element and widen.
    4264             :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    4265           0 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    4266           0 :   unsigned Increment = LdEltVT.getSizeInBits() / 8;
    4267           0 :   Ops[0] =
    4268           0 :       DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
    4269           0 :                      LdEltVT, Align, MMOFlags, AAInfo);
    4270           0 :   LdChain.push_back(Ops[0].getValue(1));
    4271             :   unsigned i = 0, Offset = Increment;
    4272           0 :   for (i=1; i < NumElts; ++i, Offset += Increment) {
    4273           0 :     SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
    4274           0 :     Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
    4275           0 :                             LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
    4276           0 :                             Align, MMOFlags, AAInfo);
    4277           0 :     LdChain.push_back(Ops[i].getValue(1));
    4278             :   }
    4279             : 
    4280             :   // Fill the rest with undefs.
    4281           0 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    4282           0 :   for (; i != WidenNumElts; ++i)
    4283           0 :     Ops[i] = UndefVal;
    4284             : 
    4285           0 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    4286             : }
    4287             : 
    4288         908 : void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
    4289             :                                             StoreSDNode *ST) {
    4290             :   // The strategy assumes that we can efficiently store power-of-two widths.
    4291             :   // The routine chops the vector into the largest vector stores with the same
    4292             :   // element type or scalar stores.
    4293         908 :   SDValue  Chain = ST->getChain();
    4294         908 :   SDValue  BasePtr = ST->getBasePtr();
    4295         908 :   unsigned Align = ST->getAlignment();
    4296         908 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
    4297             :   AAMDNodes AAInfo = ST->getAAInfo();
    4298         908 :   SDValue  ValOp = GetWidenedVector(ST->getValue());
    4299             :   SDLoc dl(ST);
    4300             : 
    4301         908 :   EVT StVT = ST->getMemoryVT();
    4302         908 :   unsigned StWidth = StVT.getSizeInBits();
    4303         908 :   EVT ValVT = ValOp.getValueType();
    4304         908 :   unsigned ValWidth = ValVT.getSizeInBits();
    4305         908 :   EVT ValEltVT = ValVT.getVectorElementType();
    4306         908 :   unsigned ValEltWidth = ValEltVT.getSizeInBits();
    4307             :   assert(StVT.getVectorElementType() == ValEltVT);
    4308             : 
    4309             :   int Idx = 0;          // current index to store
    4310             :   unsigned Offset = 0;  // offset from base to store
    4311        2428 :   while (StWidth != 0) {
    4312             :     // Find the largest vector type we can store with.
    4313        1520 :     EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
    4314        1520 :     unsigned NewVTWidth = NewVT.getSizeInBits();
    4315        1520 :     unsigned Increment = NewVTWidth / 8;
    4316        1520 :     if (NewVT.isVector()) {
    4317             :       unsigned NumVTElts = NewVT.getVectorNumElements();
    4318             :       do {
    4319         989 :         SDValue EOp = DAG.getNode(
    4320             :             ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
    4321         989 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4322         989 :         StChain.push_back(DAG.getStore(
    4323         989 :             Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
    4324        2967 :             MinAlign(Align, Offset), MMOFlags, AAInfo));
    4325         989 :         StWidth -= NewVTWidth;
    4326         989 :         Offset += Increment;
    4327         989 :         Idx += NumVTElts;
    4328             : 
    4329         989 :         BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
    4330         989 :       } while (StWidth != 0 && StWidth >= NewVTWidth);
    4331             :     } else {
    4332             :       // Cast the vector to the scalar type we can store.
    4333        1213 :       unsigned NumElts = ValWidth / NewVTWidth;
    4334        1213 :       EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
    4335        2426 :       SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
    4336             :       // Readjust index position based on new vector type.
    4337        1213 :       Idx = Idx * ValEltWidth / NewVTWidth;
    4338             :       do {
    4339        1404 :         SDValue EOp = DAG.getNode(
    4340             :             ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
    4341        1404 :             DAG.getConstant(Idx++, dl,
    4342        1404 :                             TLI.getVectorIdxTy(DAG.getDataLayout())));
    4343        1404 :         StChain.push_back(DAG.getStore(
    4344        1404 :             Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
    4345        4212 :             MinAlign(Align, Offset), MMOFlags, AAInfo));
    4346        1404 :         StWidth -= NewVTWidth;
    4347        1404 :         Offset += Increment;
    4348        1404 :         BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
    4349        1404 :       } while (StWidth != 0 && StWidth >= NewVTWidth);
    4350             :       // Restore index back to be relative to the original widen element type.
    4351        1213 :       Idx = Idx * NewVTWidth / ValEltWidth;
    4352             :     }
    4353             :   }
    4354         908 : }
    4355             : 
    4356             : void
    4357           0 : DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
    4358             :                                             StoreSDNode *ST) {
    4359             :   // For extension loads, it may not be more efficient to truncate the vector
    4360             :   // and then store it. Instead, we extract each element and then store it.
    4361           0 :   SDValue Chain = ST->getChain();
    4362           0 :   SDValue BasePtr = ST->getBasePtr();
    4363           0 :   unsigned Align = ST->getAlignment();
    4364           0 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
    4365             :   AAMDNodes AAInfo = ST->getAAInfo();
    4366           0 :   SDValue ValOp = GetWidenedVector(ST->getValue());
    4367             :   SDLoc dl(ST);
    4368             : 
    4369           0 :   EVT StVT = ST->getMemoryVT();
    4370           0 :   EVT ValVT = ValOp.getValueType();
    4371             : 
    4372             :   // It must be true that the wide vector type is bigger than where we need to
    4373             :   // store.
    4374             :   assert(StVT.isVector() && ValOp.getValueType().isVector());
    4375             :   assert(StVT.bitsLT(ValOp.getValueType()));
    4376             : 
    4377             :   // For truncating stores, we can not play the tricks of chopping legal vector
    4378             :   // types and bitcast it to the right type. Instead, we unroll the store.
    4379           0 :   EVT StEltVT  = StVT.getVectorElementType();
    4380           0 :   EVT ValEltVT = ValVT.getVectorElementType();
    4381           0 :   unsigned Increment = ValEltVT.getSizeInBits() / 8;
    4382             :   unsigned NumElts = StVT.getVectorNumElements();
    4383           0 :   SDValue EOp = DAG.getNode(
    4384             :       ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
    4385           0 :       DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4386           0 :   StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
    4387           0 :                                       ST->getPointerInfo(), StEltVT, Align,
    4388           0 :                                       MMOFlags, AAInfo));
    4389             :   unsigned Offset = Increment;
    4390           0 :   for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
    4391           0 :     SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
    4392           0 :     SDValue EOp = DAG.getNode(
    4393             :         ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
    4394           0 :         DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4395           0 :     StChain.push_back(DAG.getTruncStore(
    4396           0 :         Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
    4397           0 :         StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
    4398             :   }
    4399           0 : }
    4400             : 
    4401             : /// Modifies a vector input (widen or narrows) to a vector of NVT.  The
    4402             : /// input vector must have the same element type as NVT.
    4403             : /// FillWithZeroes specifies that the vector should be widened with zeroes.
    4404          76 : SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
    4405             :                                        bool FillWithZeroes) {
    4406             :   // Note that InOp might have been widened so it might already have
    4407             :   // the right width or it might need be narrowed.
    4408          76 :   EVT InVT = InOp.getValueType();
    4409             :   assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
    4410             :          "input and widen element type must match");
    4411             :   SDLoc dl(InOp);
    4412             : 
    4413             :   // Check if InOp already has the right width.
    4414          76 :   if (InVT == NVT)
    4415           0 :     return InOp;
    4416             : 
    4417          76 :   unsigned InNumElts = InVT.getVectorNumElements();
    4418          76 :   unsigned WidenNumElts = NVT.getVectorNumElements();
    4419          76 :   if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
    4420          72 :     unsigned NumConcat = WidenNumElts / InNumElts;
    4421          72 :     SmallVector<SDValue, 16> Ops(NumConcat);
    4422          37 :     SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
    4423          72 :       DAG.getUNDEF(InVT);
    4424          72 :     Ops[0] = InOp;
    4425         144 :     for (unsigned i = 1; i != NumConcat; ++i)
    4426         144 :       Ops[i] = FillVal;
    4427             : 
    4428         144 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
    4429             :   }
    4430             : 
    4431           4 :   if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
    4432           0 :     return DAG.getNode(
    4433             :         ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
    4434           0 :         DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4435             : 
    4436             :   // Fall back to extract and build.
    4437           4 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    4438           4 :   EVT EltVT = NVT.getVectorElementType();
    4439           4 :   unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
    4440             :   unsigned Idx;
    4441          16 :   for (Idx = 0; Idx < MinNumElts; ++Idx)
    4442          12 :     Ops[Idx] = DAG.getNode(
    4443             :         ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    4444          12 :         DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4445             : 
    4446           4 :   SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
    4447           4 :     DAG.getUNDEF(EltVT);
    4448           8 :   for ( ; Idx < WidenNumElts; ++Idx)
    4449           8 :     Ops[Idx] = FillVal;
    4450           8 :   return DAG.getBuildVector(NVT, dl, Ops);
    4451             : }

Generated by: LCOV version 1.13