LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - LegalizeVectorTypes.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1861 2101 88.6 %
Date: 2018-06-17 00:07:59 Functions: 107 116 92.2 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file performs vector type splitting and scalarization for LegalizeTypes.
      11             : // Scalarization is the act of changing a computation in an illegal one-element
      12             : // vector type to be a computation in its scalar element type.  For example,
      13             : // implementing <1 x f32> arithmetic in a scalar f32 register.  This is needed
      14             : // as a base case when scalarizing vector arithmetic like <4 x f32>, which
      15             : // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
      16             : // types.
      17             : // Splitting is the act of changing a computation in an invalid vector type to
      18             : // be a computation in two vectors of half the size.  For example, implementing
      19             : // <128 x f32> operations in terms of two <64 x f32> operations.
      20             : //
      21             : //===----------------------------------------------------------------------===//
      22             : 
      23             : #include "LegalizeTypes.h"
      24             : #include "llvm/IR/DataLayout.h"
      25             : #include "llvm/Support/ErrorHandling.h"
      26             : #include "llvm/Support/raw_ostream.h"
      27             : using namespace llvm;
      28             : 
      29             : #define DEBUG_TYPE "legalize-types"
      30             : 
      31             : //===----------------------------------------------------------------------===//
      32             : //  Result Vector Scalarization: <1 x ty> -> ty.
      33             : //===----------------------------------------------------------------------===//
      34             : 
      35       56000 : void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
      36             :   LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
      37             :              dbgs() << "\n");
      38       56000 :   SDValue R = SDValue();
      39             : 
      40      112000 :   switch (N->getOpcode()) {
      41           0 :   default:
      42             : #ifndef NDEBUG
      43             :     dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
      44             :     N->dump(&DAG);
      45             :     dbgs() << "\n";
      46             : #endif
      47           0 :     report_fatal_error("Do not know how to scalarize the result of this "
      48             :                        "operator!\n");
      49             : 
      50           0 :   case ISD::MERGE_VALUES:      R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
      51       16143 :   case ISD::BITCAST:           R = ScalarizeVecRes_BITCAST(N); break;
      52        7243 :   case ISD::BUILD_VECTOR:      R = ScalarizeVecRes_BUILD_VECTOR(N); break;
      53        4370 :   case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
      54         754 :   case ISD::FP_ROUND:          R = ScalarizeVecRes_FP_ROUND(N); break;
      55           0 :   case ISD::FP_ROUND_INREG:    R = ScalarizeVecRes_InregOp(N); break;
      56           0 :   case ISD::FPOWI:             R = ScalarizeVecRes_FPOWI(N); break;
      57         148 :   case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
      58        4889 :   case ISD::LOAD:           R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
      59         149 :   case ISD::SCALAR_TO_VECTOR:  R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
      60          19 :   case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
      61         175 :   case ISD::VSELECT:           R = ScalarizeVecRes_VSELECT(N); break;
      62         197 :   case ISD::SELECT:            R = ScalarizeVecRes_SELECT(N); break;
      63           0 :   case ISD::SELECT_CC:         R = ScalarizeVecRes_SELECT_CC(N); break;
      64         402 :   case ISD::SETCC:             R = ScalarizeVecRes_SETCC(N); break;
      65         128 :   case ISD::UNDEF:             R = ScalarizeVecRes_UNDEF(N); break;
      66           0 :   case ISD::VECTOR_SHUFFLE:    R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
      67          20 :   case ISD::ANY_EXTEND_VECTOR_INREG:
      68             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
      69             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
      70          20 :     R = ScalarizeVecRes_VecInregOp(N);
      71          20 :     break;
      72       17065 :   case ISD::ANY_EXTEND:
      73             :   case ISD::BITREVERSE:
      74             :   case ISD::BSWAP:
      75             :   case ISD::CTLZ:
      76             :   case ISD::CTLZ_ZERO_UNDEF:
      77             :   case ISD::CTPOP:
      78             :   case ISD::CTTZ:
      79             :   case ISD::CTTZ_ZERO_UNDEF:
      80             :   case ISD::FABS:
      81             :   case ISD::FCEIL:
      82             :   case ISD::FCOS:
      83             :   case ISD::FEXP:
      84             :   case ISD::FEXP2:
      85             :   case ISD::FFLOOR:
      86             :   case ISD::FLOG:
      87             :   case ISD::FLOG10:
      88             :   case ISD::FLOG2:
      89             :   case ISD::FNEARBYINT:
      90             :   case ISD::FNEG:
      91             :   case ISD::FP_EXTEND:
      92             :   case ISD::FP_TO_SINT:
      93             :   case ISD::FP_TO_UINT:
      94             :   case ISD::FRINT:
      95             :   case ISD::FROUND:
      96             :   case ISD::FSIN:
      97             :   case ISD::FSQRT:
      98             :   case ISD::FTRUNC:
      99             :   case ISD::SIGN_EXTEND:
     100             :   case ISD::SINT_TO_FP:
     101             :   case ISD::TRUNCATE:
     102             :   case ISD::UINT_TO_FP:
     103             :   case ISD::ZERO_EXTEND:
     104             :   case ISD::FCANONICALIZE:
     105       17065 :     R = ScalarizeVecRes_UnaryOp(N);
     106       17065 :     break;
     107             : 
     108        4284 :   case ISD::ADD:
     109             :   case ISD::AND:
     110             :   case ISD::FADD:
     111             :   case ISD::FCOPYSIGN:
     112             :   case ISD::FDIV:
     113             :   case ISD::FMUL:
     114             :   case ISD::FMINNUM:
     115             :   case ISD::FMAXNUM:
     116             :   case ISD::FMINNAN:
     117             :   case ISD::FMAXNAN:
     118             :   case ISD::SMIN:
     119             :   case ISD::SMAX:
     120             :   case ISD::UMIN:
     121             :   case ISD::UMAX:
     122             : 
     123             :   case ISD::FPOW:
     124             :   case ISD::FREM:
     125             :   case ISD::FSUB:
     126             :   case ISD::MUL:
     127             :   case ISD::OR:
     128             :   case ISD::SDIV:
     129             :   case ISD::SREM:
     130             :   case ISD::SUB:
     131             :   case ISD::UDIV:
     132             :   case ISD::UREM:
     133             :   case ISD::XOR:
     134             :   case ISD::SHL:
     135             :   case ISD::SRA:
     136             :   case ISD::SRL:
     137        4284 :     R = ScalarizeVecRes_BinOp(N);
     138        4284 :     break;
     139          14 :   case ISD::FMA:
     140          14 :     R = ScalarizeVecRes_TernaryOp(N);
     141          14 :     break;
     142             :   }
     143             : 
     144             :   // If R is null, the sub-method took care of registering the result.
     145       56000 :   if (R.getNode())
     146       56000 :     SetScalarizedVector(SDValue(N, ResNo), R);
     147       56000 : }
     148             : 
     149        4284 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
     150        8568 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     151        8568 :   SDValue RHS = GetScalarizedVector(N->getOperand(1));
     152        8568 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     153       17136 :                      LHS.getValueType(), LHS, RHS, N->getFlags());
     154             : }
     155             : 
     156          14 : SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
     157          28 :   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
     158          28 :   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
     159          28 :   SDValue Op2 = GetScalarizedVector(N->getOperand(2));
     160          28 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
     161          56 :                      Op0.getValueType(), Op0, Op1, Op2);
     162             : }
     163             : 
     164           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
     165             :                                                        unsigned ResNo) {
     166           0 :   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
     167           0 :   return GetScalarizedVector(Op);
     168             : }
     169             : 
     170       16143 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
     171       16143 :   SDValue Op = N->getOperand(0);
     172       32286 :   if (Op.getValueType().isVector()
     173       16387 :       && Op.getValueType().getVectorNumElements() == 1
     174       16143 :       && !isSimpleLegalType(Op.getValueType()))
     175         123 :     Op = GetScalarizedVector(Op);
     176       32286 :   EVT NewVT = N->getValueType(0).getVectorElementType();
     177       32286 :   return DAG.getNode(ISD::BITCAST, SDLoc(N),
     178       32286 :                      NewVT, Op);
     179             : }
     180             : 
     181        7243 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
     182       14486 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     183        7243 :   SDValue InOp = N->getOperand(0);
     184             :   // The BUILD_VECTOR operands may be of wider element types and
     185             :   // we may need to truncate them back to the requested return type.
     186        7243 :   if (EltVT.isInteger())
     187       17538 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
     188        1397 :   return InOp;
     189             : }
     190             : 
     191        4370 : SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
     192        8740 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
     193       13110 :                      N->getValueType(0).getVectorElementType(),
     194       17480 :                      N->getOperand(0), N->getOperand(1));
     195             : }
     196             : 
     197         754 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
     198        1508 :   EVT NewVT = N->getValueType(0).getVectorElementType();
     199        1508 :   SDValue Op = GetScalarizedVector(N->getOperand(0));
     200        1508 :   return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
     201        2262 :                      NewVT, Op, N->getOperand(1));
     202             : }
     203             : 
     204           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
     205           0 :   SDValue Op = GetScalarizedVector(N->getOperand(0));
     206           0 :   return DAG.getNode(ISD::FPOWI, SDLoc(N),
     207           0 :                      Op.getValueType(), Op, N->getOperand(1));
     208             : }
     209             : 
     210         148 : SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
     211             :   // The value to insert may have a wider type than the vector element type,
     212             :   // so be sure to truncate it to the element type if necessary.
     213         148 :   SDValue Op = N->getOperand(1);
     214         296 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     215         172 :   if (Op.getValueType() != EltVT)
     216             :     // FIXME: Can this happen for floating point types?
     217           0 :     Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
     218         148 :   return Op;
     219             : }
     220             : 
     221        4889 : SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
     222             :   assert(N->isUnindexed() && "Indexed vector load?");
     223             : 
     224        4889 :   SDValue Result = DAG.getLoad(
     225             :       ISD::UNINDEXED, N->getExtensionType(),
     226       19556 :       N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
     227        4889 :       N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
     228       14667 :       N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
     229        4889 :       N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
     230       39112 :       N->getAAInfo());
     231             : 
     232             :   // Legalize the chain result - switch anything that used the old chain to
     233             :   // use the new one.
     234        4889 :   ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
     235        4889 :   return Result;
     236             : }
     237             : 
     238       17065 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
     239             :   // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
     240       34130 :   EVT DestVT = N->getValueType(0).getVectorElementType();
     241       17065 :   SDValue Op = N->getOperand(0);
     242       17065 :   EVT OpVT = Op.getValueType();
     243             :   SDLoc DL(N);
     244             :   // The result needs scalarizing, but it's not a given that the source does.
     245             :   // This is a workaround for targets where it's impossible to scalarize the
     246             :   // result of a conversion, because the source type is legal.
     247             :   // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
     248             :   // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
     249             :   // legal and was not scalarized.
     250             :   // See the similar logic in ScalarizeVecRes_SETCC
     251       34130 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     252       17059 :     Op = GetScalarizedVector(Op);
     253             :   } else {
     254           6 :     EVT VT = OpVT.getVectorElementType();
     255          12 :     Op = DAG.getNode(
     256             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
     257          24 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     258             :   }
     259       85325 :   return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
     260             : }
     261             : 
     262          19 : SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
     263          38 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     264          38 :   EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
     265          38 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     266          38 :   return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
     267          76 :                      LHS, DAG.getValueType(ExtVT));
     268             : }
     269             : 
     270          20 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
     271             :   SDLoc DL(N);
     272          20 :   SDValue Op = N->getOperand(0);
     273             : 
     274          20 :   EVT OpVT = Op.getValueType();
     275          20 :   EVT OpEltVT = OpVT.getVectorElementType();
     276          40 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     277             : 
     278          40 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     279           0 :     Op = GetScalarizedVector(Op);
     280             :   } else {
     281          40 :     Op = DAG.getNode(
     282             :         ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
     283          80 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     284             :   }
     285             : 
     286          40 :   switch (N->getOpcode()) {
     287          20 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     288          40 :     return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
     289           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     290           0 :     return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
     291           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     292           0 :     return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
     293             :   }
     294             : 
     295           0 :   llvm_unreachable("Illegal extend_vector_inreg opcode");
     296             : }
     297             : 
     298         149 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
     299             :   // If the operand is wider than the vector element type then it is implicitly
     300             :   // truncated.  Make that explicit here.
     301         298 :   EVT EltVT = N->getValueType(0).getVectorElementType();
     302         149 :   SDValue InOp = N->getOperand(0);
     303         157 :   if (InOp.getValueType() != EltVT)
     304           0 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
     305         149 :   return InOp;
     306             : }
     307             : 
     308         175 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
     309         175 :   SDValue Cond = N->getOperand(0);
     310         175 :   EVT OpVT = Cond.getValueType();
     311             :   SDLoc DL(N);
     312             :   // The vselect result and true/value operands needs scalarizing, but it's
     313             :   // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
     314             :   // See the similar logic in ScalarizeVecRes_SETCC
     315         350 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     316         157 :     Cond = GetScalarizedVector(Cond);
     317             :   } else {
     318          18 :     EVT VT = OpVT.getVectorElementType();
     319          36 :     Cond = DAG.getNode(
     320             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
     321          72 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     322             :   }
     323             : 
     324         350 :   SDValue LHS = GetScalarizedVector(N->getOperand(1));
     325             :   TargetLowering::BooleanContent ScalarBool =
     326         175 :       TLI.getBooleanContents(false, false);
     327             :   TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
     328             : 
     329             :   // If integer and float booleans have different contents then we can't
     330             :   // reliably optimize in all cases. There is a full explanation for this in
     331             :   // DAGCombiner::visitSELECT() where the same issue affects folding
     332             :   // (select C, 0, 1) to (xor C, 1).
     333         175 :   if (TLI.getBooleanContents(false, false) !=
     334             :       TLI.getBooleanContents(false, true)) {
     335             :     // At least try the common case where the boolean is generated by a
     336             :     // comparison.
     337           0 :     if (Cond->getOpcode() == ISD::SETCC) {
     338           0 :       EVT OpVT = Cond->getOperand(0).getValueType();
     339           0 :       ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
     340           0 :       VecBool = TLI.getBooleanContents(OpVT);
     341             :     } else
     342             :       ScalarBool = TargetLowering::UndefinedBooleanContent;
     343             :   }
     344             : 
     345         175 :   EVT CondVT = Cond.getValueType();
     346         175 :   if (ScalarBool != VecBool) {
     347          49 :     switch (ScalarBool) {
     348             :       case TargetLowering::UndefinedBooleanContent:
     349             :         break;
     350          49 :       case TargetLowering::ZeroOrOneBooleanContent:
     351             :         assert(VecBool == TargetLowering::UndefinedBooleanContent ||
     352             :                VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
     353             :         // Vector read from all ones, scalar expects a single 1 so mask.
     354         147 :         Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
     355          98 :                            Cond, DAG.getConstant(1, SDLoc(N), CondVT));
     356          49 :         break;
     357           0 :       case TargetLowering::ZeroOrNegativeOneBooleanContent:
     358             :         assert(VecBool == TargetLowering::UndefinedBooleanContent ||
     359             :                VecBool == TargetLowering::ZeroOrOneBooleanContent);
     360             :         // Vector reads from a one, scalar from all ones so sign extend.
     361           0 :         Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
     362           0 :                            Cond, DAG.getValueType(MVT::i1));
     363           0 :         break;
     364             :     }
     365             :   }
     366             : 
     367             :   // Truncate the condition if needed
     368         175 :   auto BoolVT = getSetCCResultType(CondVT);
     369         175 :   if (BoolVT.bitsLT(CondVT))
     370          12 :     Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
     371             : 
     372         350 :   return DAG.getSelect(SDLoc(N),
     373             :                        LHS.getValueType(), Cond, LHS,
     374         700 :                        GetScalarizedVector(N->getOperand(2)));
     375             : }
     376             : 
     377         197 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
     378         394 :   SDValue LHS = GetScalarizedVector(N->getOperand(1));
     379         394 :   return DAG.getSelect(SDLoc(N),
     380         197 :                        LHS.getValueType(), N->getOperand(0), LHS,
     381         985 :                        GetScalarizedVector(N->getOperand(2)));
     382             : }
     383             : 
     384           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
     385           0 :   SDValue LHS = GetScalarizedVector(N->getOperand(2));
     386           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
     387           0 :                      N->getOperand(0), N->getOperand(1),
     388             :                      LHS, GetScalarizedVector(N->getOperand(3)),
     389           0 :                      N->getOperand(4));
     390             : }
     391             : 
     392         128 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
     393         256 :   return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
     394             : }
     395             : 
     396           0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
     397             :   // Figure out if the scalar is the LHS or RHS and return it.
     398           0 :   SDValue Arg = N->getOperand(2).getOperand(0);
     399           0 :   if (Arg.isUndef())
     400           0 :     return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
     401           0 :   unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
     402           0 :   return GetScalarizedVector(N->getOperand(Op));
     403             : }
     404             : 
     405         402 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
     406             :   assert(N->getValueType(0).isVector() &&
     407             :          N->getOperand(0).getValueType().isVector() &&
     408             :          "Operand types must be vectors");
     409         402 :   SDValue LHS = N->getOperand(0);
     410         402 :   SDValue RHS = N->getOperand(1);
     411         402 :   EVT OpVT = LHS.getValueType();
     412         804 :   EVT NVT = N->getValueType(0).getVectorElementType();
     413             :   SDLoc DL(N);
     414             : 
     415             :   // The result needs scalarizing, but it's not a given that the source does.
     416         804 :   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     417         396 :     LHS = GetScalarizedVector(LHS);
     418         396 :     RHS = GetScalarizedVector(RHS);
     419             :   } else {
     420           6 :     EVT VT = OpVT.getVectorElementType();
     421          12 :     LHS = DAG.getNode(
     422             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
     423          24 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     424          12 :     RHS = DAG.getNode(
     425             :         ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
     426          24 :         DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
     427             :   }
     428             : 
     429             :   // Turn it into a scalar SETCC.
     430         402 :   SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
     431         804 :                             N->getOperand(2));
     432             :   // Vectors may have a different boolean contents to scalars.  Promote the
     433             :   // value appropriately.
     434             :   ISD::NodeType ExtendCode =
     435         402 :       TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
     436        1206 :   return DAG.getNode(ExtendCode, DL, NVT, Res);
     437             : }
     438             : 
     439             : 
     440             : //===----------------------------------------------------------------------===//
     441             : //  Operand Vector Scalarization <1 x ty> -> ty.
     442             : //===----------------------------------------------------------------------===//
     443             : 
     444       16102 : bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
     445             :   LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
     446             :              dbgs() << "\n");
     447       16102 :   SDValue Res = SDValue();
     448             : 
     449             :   if (!Res.getNode()) {
     450       32204 :     switch (N->getOpcode()) {
     451           0 :     default:
     452             : #ifndef NDEBUG
     453             :       dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
     454             :       N->dump(&DAG);
     455             :       dbgs() << "\n";
     456             : #endif
     457           0 :       report_fatal_error("Do not know how to scalarize this operator's "
     458             :                          "operand!\n");
     459        1979 :     case ISD::BITCAST:
     460        1979 :       Res = ScalarizeVecOp_BITCAST(N);
     461        1979 :       break;
     462          22 :     case ISD::ANY_EXTEND:
     463             :     case ISD::ZERO_EXTEND:
     464             :     case ISD::SIGN_EXTEND:
     465             :     case ISD::TRUNCATE:
     466             :     case ISD::FP_TO_SINT:
     467             :     case ISD::FP_TO_UINT:
     468             :     case ISD::SINT_TO_FP:
     469             :     case ISD::UINT_TO_FP:
     470          22 :       Res = ScalarizeVecOp_UnaryOp(N);
     471          22 :       break;
     472        6023 :     case ISD::CONCAT_VECTORS:
     473        6023 :       Res = ScalarizeVecOp_CONCAT_VECTORS(N);
     474        6023 :       break;
     475        4905 :     case ISD::EXTRACT_VECTOR_ELT:
     476        4905 :       Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
     477        4905 :       break;
     478           1 :     case ISD::VSELECT:
     479           1 :       Res = ScalarizeVecOp_VSELECT(N);
     480           1 :       break;
     481          14 :     case ISD::SETCC:
     482          14 :       Res = ScalarizeVecOp_VSETCC(N);
     483          14 :       break;
     484             :     case ISD::STORE:
     485        3155 :       Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
     486        3155 :       break;
     487           3 :     case ISD::FP_ROUND:
     488           3 :       Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
     489           3 :       break;
     490             :     }
     491             :   }
     492             : 
     493             :   // If the result is null, the sub-method took care of registering results etc.
     494       16102 :   if (!Res.getNode()) return false;
     495             : 
     496             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
     497             :   // core about this.
     498       16102 :   if (Res.getNode() == N)
     499             :     return true;
     500             : 
     501             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
     502             :          "Invalid operand expansion");
     503             : 
     504       16102 :   ReplaceValueWith(SDValue(N, 0), Res);
     505       16102 :   return false;
     506             : }
     507             : 
     508             : /// If the value to convert is a vector that needs to be scalarized, it must be
     509             : /// <1 x ty>. Convert the element instead.
     510        1979 : SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
     511        3958 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     512        3958 :   return DAG.getNode(ISD::BITCAST, SDLoc(N),
     513        5937 :                      N->getValueType(0), Elt);
     514             : }
     515             : 
     516             : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
     517             : /// Do the operation on the element instead.
     518          22 : SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
     519             :   assert(N->getValueType(0).getVectorNumElements() == 1 &&
     520             :          "Unexpected vector type!");
     521          44 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     522          44 :   SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
     523         110 :                            N->getValueType(0).getScalarType(), Elt);
     524             :   // Revectorize the result so the types line up with what the uses of this
     525             :   // expression expect.
     526          88 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
     527             : }
     528             : 
     529             : /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
     530        6023 : SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
     531       12046 :   SmallVector<SDValue, 8> Ops(N->getNumOperands());
     532       36162 :   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
     533       36174 :     Ops[i] = GetScalarizedVector(N->getOperand(i));
     534       30115 :   return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
     535             : }
     536             : 
     537             : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
     538             : /// so just return the element, ignoring the index.
     539        4905 : SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
     540        9810 :   EVT VT = N->getValueType(0);
     541        9810 :   SDValue Res = GetScalarizedVector(N->getOperand(0));
     542           0 :   if (Res.getValueType() != VT)
     543         760 :     Res = VT.isFloatingPoint()
     544        1900 :               ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
     545        1137 :               : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
     546        4905 :   return Res;
     547             : }
     548             : 
     549             : /// If the input condition is a vector that needs to be scalarized, it must be
     550             : /// <1 x i1>, so just convert to a normal ISD::SELECT
     551             : /// (still with vector output type since that was acceptable if we got here).
     552           1 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
     553           2 :   SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
     554           2 :   EVT VT = N->getValueType(0);
     555             : 
     556           2 :   return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
     557           3 :                      N->getOperand(2));
     558             : }
     559             : 
     560             : /// If the operand is a vector that needs to be scalarized then the
     561             : /// result must be v1i1, so just convert to a scalar SETCC and wrap
     562             : /// with a scalar_to_vector since the res type is legal if we got here
     563          14 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
     564             :   assert(N->getValueType(0).isVector() &&
     565             :          N->getOperand(0).getValueType().isVector() &&
     566             :          "Operand types must be vectors");
     567             :   assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
     568             : 
     569          28 :   EVT VT = N->getValueType(0);
     570          28 :   SDValue LHS = GetScalarizedVector(N->getOperand(0));
     571          28 :   SDValue RHS = GetScalarizedVector(N->getOperand(1));
     572             : 
     573          28 :   EVT OpVT = N->getOperand(0).getValueType();
     574          14 :   EVT NVT = VT.getVectorElementType();
     575             :   SDLoc DL(N);
     576             :   // Turn it into a scalar SETCC.
     577          14 :   SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
     578          28 :       N->getOperand(2));
     579             : 
     580             :   // Vectors may have a different boolean contents to scalars.  Promote the
     581             :   // value appropriately.
     582             :   ISD::NodeType ExtendCode =
     583          14 :       TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
     584             : 
     585          28 :   Res = DAG.getNode(ExtendCode, DL, NVT, Res);
     586             : 
     587          42 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
     588             : }
     589             : 
     590             : /// If the value to store is a vector that needs to be scalarized, it must be
     591             : /// <1 x ty>. Just store the element.
     592        3155 : SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
     593             :   assert(N->isUnindexed() && "Indexed store of one-element vector?");
     594             :   assert(OpNo == 1 && "Do not know how to scalarize this operand!");
     595             :   SDLoc dl(N);
     596             : 
     597        3155 :   if (N->isTruncatingStore())
     598           0 :     return DAG.getTruncStore(
     599             :         N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
     600           0 :         N->getBasePtr(), N->getPointerInfo(),
     601           0 :         N->getMemoryVT().getVectorElementType(), N->getAlignment(),
     602           0 :         N->getMemOperand()->getFlags(), N->getAAInfo());
     603             : 
     604        3155 :   return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
     605             :                       N->getBasePtr(), N->getPointerInfo(),
     606        3155 :                       N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
     607       12620 :                       N->getAAInfo());
     608             : }
     609             : 
     610             : /// If the value to round is a vector that needs to be scalarized, it must be
     611             : /// <1 x ty>. Convert the element instead.
     612           3 : SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
     613           6 :   SDValue Elt = GetScalarizedVector(N->getOperand(0));
     614           6 :   SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
     615           9 :                             N->getValueType(0).getVectorElementType(), Elt,
     616          12 :                             N->getOperand(1));
     617          12 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
     618             : }
     619             : 
     620             : //===----------------------------------------------------------------------===//
     621             : //  Result Vector Splitting
     622             : //===----------------------------------------------------------------------===//
     623             : 
     624             : /// This method is called when the specified result of the specified node is
     625             : /// found to need vector splitting. At this point, the node may also have
     626             : /// invalid operands or may have other results that need legalization, we just
     627             : /// know that (at least) one result needs vector splitting.
     628       82587 : void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
     629             :   LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
     630       82587 :   SDValue Lo, Hi;
     631             : 
     632             :   // See if the target wants to custom expand this node.
     633      165174 :   if (CustomLowerNode(N, N->getValueType(ResNo), true))
     634         363 :     return;
     635             : 
     636      164448 :   switch (N->getOpcode()) {
     637           0 :   default:
     638             : #ifndef NDEBUG
     639             :     dbgs() << "SplitVectorResult #" << ResNo << ": ";
     640             :     N->dump(&DAG);
     641             :     dbgs() << "\n";
     642             : #endif
     643           0 :     report_fatal_error("Do not know how to split the result of this "
     644             :                        "operator!\n");
     645             : 
     646           0 :   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
     647        1121 :   case ISD::VSELECT:
     648        1121 :   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
     649           0 :   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
     650        1724 :   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
     651       18121 :   case ISD::BITCAST:           SplitVecRes_BITCAST(N, Lo, Hi); break;
     652        8424 :   case ISD::BUILD_VECTOR:      SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
     653       15250 :   case ISD::CONCAT_VECTORS:    SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
     654        1452 :   case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
     655          13 :   case ISD::INSERT_SUBVECTOR:  SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
     656           0 :   case ISD::FP_ROUND_INREG:    SplitVecRes_InregOp(N, Lo, Hi); break;
     657           8 :   case ISD::FPOWI:             SplitVecRes_FPOWI(N, Lo, Hi); break;
     658          11 :   case ISD::FCOPYSIGN:         SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
     659         752 :   case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
     660         190 :   case ISD::SCALAR_TO_VECTOR:  SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
     661          79 :   case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
     662             :   case ISD::LOAD:
     663        3804 :     SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
     664        3804 :     break;
     665             :   case ISD::MLOAD:
     666          20 :     SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
     667          20 :     break;
     668             :   case ISD::MGATHER:
     669          16 :     SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
     670          16 :     break;
     671        1838 :   case ISD::SETCC:
     672        1838 :     SplitVecRes_SETCC(N, Lo, Hi);
     673        1838 :     break;
     674             :   case ISD::VECTOR_SHUFFLE:
     675        4838 :     SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
     676        4838 :     break;
     677             : 
     678         410 :   case ISD::ANY_EXTEND_VECTOR_INREG:
     679             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
     680             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
     681         410 :     SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
     682         410 :     break;
     683             : 
     684        2971 :   case ISD::BITREVERSE:
     685             :   case ISD::BSWAP:
     686             :   case ISD::CTLZ:
     687             :   case ISD::CTTZ:
     688             :   case ISD::CTLZ_ZERO_UNDEF:
     689             :   case ISD::CTTZ_ZERO_UNDEF:
     690             :   case ISD::CTPOP:
     691             :   case ISD::FABS:
     692             :   case ISD::FCEIL:
     693             :   case ISD::FCOS:
     694             :   case ISD::FEXP:
     695             :   case ISD::FEXP2:
     696             :   case ISD::FFLOOR:
     697             :   case ISD::FLOG:
     698             :   case ISD::FLOG10:
     699             :   case ISD::FLOG2:
     700             :   case ISD::FNEARBYINT:
     701             :   case ISD::FNEG:
     702             :   case ISD::FP_EXTEND:
     703             :   case ISD::FP_ROUND:
     704             :   case ISD::FP_TO_SINT:
     705             :   case ISD::FP_TO_UINT:
     706             :   case ISD::FRINT:
     707             :   case ISD::FROUND:
     708             :   case ISD::FSIN:
     709             :   case ISD::FSQRT:
     710             :   case ISD::FTRUNC:
     711             :   case ISD::SINT_TO_FP:
     712             :   case ISD::TRUNCATE:
     713             :   case ISD::UINT_TO_FP:
     714             :   case ISD::FCANONICALIZE:
     715        2971 :     SplitVecRes_UnaryOp(N, Lo, Hi);
     716        2971 :     break;
     717             : 
     718        7511 :   case ISD::ANY_EXTEND:
     719             :   case ISD::SIGN_EXTEND:
     720             :   case ISD::ZERO_EXTEND:
     721        7511 :     SplitVecRes_ExtendOp(N, Lo, Hi);
     722        7511 :     break;
     723             : 
     724       13506 :   case ISD::ADD:
     725             :   case ISD::SUB:
     726             :   case ISD::MUL:
     727             :   case ISD::MULHS:
     728             :   case ISD::MULHU:
     729             :   case ISD::FADD:
     730             :   case ISD::FSUB:
     731             :   case ISD::FMUL:
     732             :   case ISD::FMINNUM:
     733             :   case ISD::FMAXNUM:
     734             :   case ISD::FMINNAN:
     735             :   case ISD::FMAXNAN:
     736             :   case ISD::SDIV:
     737             :   case ISD::UDIV:
     738             :   case ISD::FDIV:
     739             :   case ISD::FPOW:
     740             :   case ISD::AND:
     741             :   case ISD::OR:
     742             :   case ISD::XOR:
     743             :   case ISD::SHL:
     744             :   case ISD::SRA:
     745             :   case ISD::SRL:
     746             :   case ISD::UREM:
     747             :   case ISD::SREM:
     748             :   case ISD::FREM:
     749             :   case ISD::SMIN:
     750             :   case ISD::SMAX:
     751             :   case ISD::UMIN:
     752             :   case ISD::UMAX:
     753       13506 :     SplitVecRes_BinOp(N, Lo, Hi);
     754       13506 :     break;
     755         165 :   case ISD::FMA:
     756         165 :     SplitVecRes_TernaryOp(N, Lo, Hi);
     757         165 :     break;
     758             :   }
     759             : 
     760             :   // If Lo/Hi is null, the sub-method took care of registering results etc.
     761       82224 :   if (Lo.getNode())
     762       82224 :     SetSplitVector(SDValue(N, ResNo), Lo, Hi);
     763             : }
     764             : 
     765       13506 : void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
     766             :                                          SDValue &Hi) {
     767       13506 :   SDValue LHSLo, LHSHi;
     768       27012 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
     769       13506 :   SDValue RHSLo, RHSHi;
     770       27012 :   GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
     771             :   SDLoc dl(N);
     772             : 
     773       13506 :   const SDNodeFlags Flags = N->getFlags();
     774       13506 :   unsigned Opcode = N->getOpcode();
     775       27012 :   Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
     776       27012 :   Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
     777       13506 : }
     778             : 
     779         165 : void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
     780             :                                              SDValue &Hi) {
     781         165 :   SDValue Op0Lo, Op0Hi;
     782         330 :   GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
     783         165 :   SDValue Op1Lo, Op1Hi;
     784         330 :   GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
     785         165 :   SDValue Op2Lo, Op2Hi;
     786         330 :   GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
     787             :   SDLoc dl(N);
     788             : 
     789         330 :   Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
     790         495 :                    Op0Lo, Op1Lo, Op2Lo);
     791         330 :   Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
     792         495 :                    Op0Hi, Op1Hi, Op2Hi);
     793         165 : }
     794             : 
     795       18121 : void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
     796             :                                            SDValue &Hi) {
     797             :   // We know the result is a vector.  The input may be either a vector or a
     798             :   // scalar value.
     799       18121 :   EVT LoVT, HiVT;
     800       54363 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     801             :   SDLoc dl(N);
     802             : 
     803       18121 :   SDValue InOp = N->getOperand(0);
     804             :   EVT InVT = InOp.getValueType();
     805             : 
     806             :   // Handle some special cases efficiently.
     807       36242 :   switch (getTypeAction(InVT)) {
     808             :   case TargetLowering::TypeLegal:
     809             :   case TargetLowering::TypePromoteInteger:
     810             :   case TargetLowering::TypePromoteFloat:
     811             :   case TargetLowering::TypeSoftenFloat:
     812             :   case TargetLowering::TypeScalarizeVector:
     813             :   case TargetLowering::TypeWidenVector:
     814             :     break;
     815        1882 :   case TargetLowering::TypeExpandInteger:
     816             :   case TargetLowering::TypeExpandFloat:
     817             :     // A scalar to vector conversion, where the scalar needs expansion.
     818             :     // If the vector is being split in two then we can just convert the
     819             :     // expanded pieces.
     820        1882 :     if (LoVT == HiVT) {
     821        1882 :       GetExpandedOp(InOp, Lo, Hi);
     822        3764 :       if (DAG.getDataLayout().isBigEndian())
     823             :         std::swap(Lo, Hi);
     824        3764 :       Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     825        3764 :       Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     826        1882 :       return;
     827             :     }
     828             :     break;
     829        2586 :   case TargetLowering::TypeSplitVector:
     830             :     // If the input is a vector that needs to be split, convert each split
     831             :     // piece of the input now.
     832        2586 :     GetSplitVector(InOp, Lo, Hi);
     833        5172 :     Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     834        5172 :     Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     835        2586 :     return;
     836             :   }
     837             : 
     838             :   // In the general case, convert the input to an integer and split it by hand.
     839       13653 :   EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
     840       13653 :   EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
     841       27306 :   if (DAG.getDataLayout().isBigEndian())
     842             :     std::swap(LoIntVT, HiIntVT);
     843             : 
     844       13653 :   SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
     845             : 
     846       27306 :   if (DAG.getDataLayout().isBigEndian())
     847             :     std::swap(Lo, Hi);
     848       27306 :   Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
     849       27306 :   Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
     850             : }
     851             : 
     852        8424 : void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
     853             :                                                 SDValue &Hi) {
     854        8424 :   EVT LoVT, HiVT;
     855             :   SDLoc dl(N);
     856       25272 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     857        8424 :   unsigned LoNumElts = LoVT.getVectorNumElements();
     858        8424 :   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
     859       16848 :   Lo = DAG.getBuildVector(LoVT, dl, LoOps);
     860             : 
     861       16848 :   SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
     862       16848 :   Hi = DAG.getBuildVector(HiVT, dl, HiOps);
     863        8424 : }
     864             : 
     865       15250 : void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
     866             :                                                   SDValue &Hi) {
     867             :   assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
     868             :   SDLoc dl(N);
     869       30500 :   unsigned NumSubvectors = N->getNumOperands() / 2;
     870       15250 :   if (NumSubvectors == 1) {
     871       10888 :     Lo = N->getOperand(0);
     872       10888 :     Hi = N->getOperand(1);
     873             :     return;
     874             :   }
     875             : 
     876             :   EVT LoVT, HiVT;
     877       13086 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     878             : 
     879        4362 :   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
     880        8724 :   Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
     881             : 
     882        8724 :   SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
     883        8724 :   Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
     884             : }
     885             : 
     886        1452 : void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
     887             :                                                      SDValue &Hi) {
     888        1452 :   SDValue Vec = N->getOperand(0);
     889        1452 :   SDValue Idx = N->getOperand(1);
     890             :   SDLoc dl(N);
     891             : 
     892        1452 :   EVT LoVT, HiVT;
     893        4356 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     894             : 
     895        2904 :   Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
     896        1452 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
     897        2904 :   Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
     898        1452 :                    DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
     899        5808 :                                    TLI.getVectorIdxTy(DAG.getDataLayout())));
     900        1452 : }
     901             : 
     902          13 : void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
     903             :                                                     SDValue &Hi) {
     904          13 :   SDValue Vec = N->getOperand(0);
     905          13 :   SDValue SubVec = N->getOperand(1);
     906          13 :   SDValue Idx = N->getOperand(2);
     907             :   SDLoc dl(N);
     908          13 :   GetSplitVector(Vec, Lo, Hi);
     909             : 
     910          13 :   EVT VecVT = Vec.getValueType();
     911          13 :   unsigned VecElems = VecVT.getVectorNumElements();
     912          13 :   unsigned SubElems = SubVec.getValueType().getVectorNumElements();
     913             : 
     914             :   // If we know the index is 0, and we know the subvector doesn't cross the
     915             :   // boundary between the halves, we can avoid spilling the vector, and insert
     916             :   // into the lower half of the split vector directly.
     917             :   // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
     918             :   // the index is constant and there is no boundary crossing. But those cases
     919             :   // don't seem to get hit in practice.
     920             :   if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
     921          26 :     unsigned IdxVal = ConstIdx->getZExtValue();
     922          13 :     if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
     923             :       EVT LoVT, HiVT;
     924          39 :       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
     925          26 :       Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
     926             :       return;
     927             :     }
     928             :   }
     929             : 
     930             :   // Spill the vector to the stack.
     931           0 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
     932             :   SDValue Store =
     933           0 :       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
     934             : 
     935             :   // Store the new subvector into the specified index.
     936           0 :   SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
     937           0 :   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
     938           0 :   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
     939           0 :   Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
     940             : 
     941             :   // Load the Lo part from the stack slot.
     942           0 :   Lo =
     943           0 :       DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
     944             : 
     945             :   // Increment the pointer to the other part.
     946           0 :   unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
     947           0 :   StackPtr =
     948           0 :       DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
     949           0 :                   DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
     950             : 
     951             :   // Load the Hi part from the stack slot.
     952           0 :   Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
     953           0 :                    MinAlign(Alignment, IncrementSize));
     954             : }
     955             : 
     956           8 : void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
     957             :                                          SDValue &Hi) {
     958             :   SDLoc dl(N);
     959          16 :   GetSplitVector(N->getOperand(0), Lo, Hi);
     960          32 :   Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
     961          32 :   Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
     962           8 : }
     963             : 
     964          11 : void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
     965             :                                              SDValue &Hi) {
     966          11 :   SDValue LHSLo, LHSHi;
     967          22 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
     968             :   SDLoc DL(N);
     969             : 
     970          11 :   SDValue RHSLo, RHSHi;
     971          11 :   SDValue RHS = N->getOperand(1);
     972          11 :   EVT RHSVT = RHS.getValueType();
     973          22 :   if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
     974           9 :     GetSplitVector(RHS, RHSLo, RHSHi);
     975             :   else
     976           8 :     std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
     977             : 
     978             : 
     979          33 :   Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
     980          33 :   Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
     981          11 : }
     982             : 
     983          79 : void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
     984             :                                            SDValue &Hi) {
     985          79 :   SDValue LHSLo, LHSHi;
     986         158 :   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
     987             :   SDLoc dl(N);
     988             : 
     989             :   EVT LoVT, HiVT;
     990             :   std::tie(LoVT, HiVT) =
     991         237 :     DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
     992             : 
     993         158 :   Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
     994         237 :                    DAG.getValueType(LoVT));
     995         158 :   Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
     996         237 :                    DAG.getValueType(HiVT));
     997          79 : }
     998             : 
     999         416 : void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
    1000             :                                                  SDValue &Hi) {
    1001         416 :   unsigned Opcode = N->getOpcode();
    1002         416 :   SDValue N0 = N->getOperand(0);
    1003             : 
    1004             :   SDLoc dl(N);
    1005         416 :   SDValue InLo, InHi;
    1006             : 
    1007         832 :   if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
    1008         414 :     GetSplitVector(N0, InLo, InHi);
    1009             :   else
    1010           4 :     std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
    1011             : 
    1012         832 :   EVT InLoVT = InLo.getValueType();
    1013         416 :   unsigned InNumElements = InLoVT.getVectorNumElements();
    1014             : 
    1015         416 :   EVT OutLoVT, OutHiVT;
    1016        1248 :   std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1017         416 :   unsigned OutNumElements = OutLoVT.getVectorNumElements();
    1018             :   assert((2 * OutNumElements) <= InNumElements &&
    1019             :          "Illegal extend vector in reg split");
    1020             : 
    1021             :   // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
    1022             :   // input vector (i.e. we only use InLo):
    1023             :   // OutLo will extend the first OutNumElements from InLo.
    1024             :   // OutHi will extend the next OutNumElements from InLo.
    1025             : 
    1026             :   // Shuffle the elements from InLo for OutHi into the bottom elements to
    1027             :   // create a 'fake' InHi.
    1028         832 :   SmallVector<int, 8> SplitHi(InNumElements, -1);
    1029        4812 :   for (unsigned i = 0; i != OutNumElements; ++i)
    1030        4396 :     SplitHi[i] = i + OutNumElements;
    1031         832 :   InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
    1032             : 
    1033         832 :   Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
    1034         832 :   Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
    1035         416 : }
    1036             : 
    1037         752 : void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
    1038             :                                                      SDValue &Hi) {
    1039         752 :   SDValue Vec = N->getOperand(0);
    1040         752 :   SDValue Elt = N->getOperand(1);
    1041         752 :   SDValue Idx = N->getOperand(2);
    1042             :   SDLoc dl(N);
    1043         752 :   GetSplitVector(Vec, Lo, Hi);
    1044             : 
    1045             :   if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
    1046        1430 :     unsigned IdxVal = CIdx->getZExtValue();
    1047        1430 :     unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
    1048         715 :     if (IdxVal < LoNumElts)
    1049         810 :       Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
    1050         405 :                        Lo.getValueType(), Lo, Elt, Idx);
    1051             :     else
    1052         310 :       Hi =
    1053         930 :           DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
    1054         310 :                       DAG.getConstant(IdxVal - LoNumElts, dl,
    1055        1240 :                                       TLI.getVectorIdxTy(DAG.getDataLayout())));
    1056             :     return;
    1057             :   }
    1058             : 
    1059             :   // See if the target wants to custom expand this node.
    1060          74 :   if (CustomLowerNode(N, N->getValueType(0), true))
    1061             :     return;
    1062             : 
    1063             :   // Make the vector elements byte-addressable if they aren't already.
    1064          37 :   EVT VecVT = Vec.getValueType();
    1065          37 :   EVT EltVT = VecVT.getVectorElementType();
    1066          37 :   if (VecVT.getScalarSizeInBits() < 8) {
    1067           8 :     EltVT = MVT::i8;
    1068           8 :     VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
    1069           8 :                              VecVT.getVectorNumElements());
    1070          16 :     Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
    1071             :     // Extend the element type to match if needed.
    1072           8 :     if (EltVT.bitsGT(Elt.getValueType()))
    1073          16 :       Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
    1074             :   }
    1075             : 
    1076             :   // Spill the vector to the stack.
    1077          37 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1078          37 :   auto &MF = DAG.getMachineFunction();
    1079          37 :   auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1080          37 :   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
    1081          74 :   SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
    1082             : 
    1083             :   // Store the new element.  This may be larger than the vector element type,
    1084             :   // so use a truncating store.
    1085          37 :   SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1086          37 :   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
    1087          74 :   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
    1088          74 :   Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
    1089          74 :                             MachinePointerInfo::getUnknownStack(MF), EltVT);
    1090             : 
    1091          37 :   EVT LoVT, HiVT;
    1092          74 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
    1093             : 
    1094             :   // Load the Lo part from the stack slot.
    1095          74 :   Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
    1096             : 
    1097             :   // Increment the pointer to the other part.
    1098          37 :   unsigned IncrementSize = LoVT.getSizeInBits() / 8;
    1099          74 :   StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
    1100             :                          DAG.getConstant(IncrementSize, dl,
    1101          74 :                                          StackPtr.getValueType()));
    1102             : 
    1103             :   // Load the Hi part from the stack slot.
    1104          74 :   Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
    1105             :                    PtrInfo.getWithOffset(IncrementSize),
    1106          74 :                    MinAlign(Alignment, IncrementSize));
    1107             : 
    1108             :   // If we adjusted the original type, we need to truncate the results.
    1109         111 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1110          82 :   if (LoVT != Lo.getValueType())
    1111          16 :     Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
    1112          82 :   if (HiVT != Hi.getValueType())
    1113          16 :     Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
    1114             : }
    1115             : 
    1116         190 : void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
    1117             :                                                     SDValue &Hi) {
    1118         190 :   EVT LoVT, HiVT;
    1119             :   SDLoc dl(N);
    1120         570 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1121         570 :   Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
    1122         190 :   Hi = DAG.getUNDEF(HiVT);
    1123         190 : }
    1124             : 
    1125        3804 : void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
    1126             :                                         SDValue &Hi) {
    1127             :   assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
    1128        3804 :   EVT LoVT, HiVT;
    1129             :   SDLoc dl(LD);
    1130       11412 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
    1131             : 
    1132             :   ISD::LoadExtType ExtType = LD->getExtensionType();
    1133        3804 :   SDValue Ch = LD->getChain();
    1134        3804 :   SDValue Ptr = LD->getBasePtr();
    1135        7608 :   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
    1136        3804 :   EVT MemoryVT = LD->getMemoryVT();
    1137        3804 :   unsigned Alignment = LD->getOriginalAlignment();
    1138        3804 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    1139             :   AAMDNodes AAInfo = LD->getAAInfo();
    1140             : 
    1141        3804 :   EVT LoMemVT, HiMemVT;
    1142        7608 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1143             : 
    1144        7608 :   Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
    1145        3804 :                    LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
    1146             : 
    1147        3804 :   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
    1148        3804 :   Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
    1149        7608 :   Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
    1150        3804 :                    LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
    1151        3804 :                    Alignment, MMOFlags, AAInfo);
    1152             : 
    1153             :   // Build a factor node to remember that this load is independent of the
    1154             :   // other one.
    1155        7608 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1156        7608 :                    Hi.getValue(1));
    1157             : 
    1158             :   // Legalize the chain result - switch anything that used the old chain to
    1159             :   // use the new one.
    1160        3804 :   ReplaceValueWith(SDValue(LD, 1), Ch);
    1161        3804 : }
    1162             : 
    1163          20 : void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
    1164             :                                          SDValue &Lo, SDValue &Hi) {
    1165          20 :   EVT LoVT, HiVT;
    1166             :   SDLoc dl(MLD);
    1167          60 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
    1168             : 
    1169          20 :   SDValue Ch = MLD->getChain();
    1170          20 :   SDValue Ptr = MLD->getBasePtr();
    1171          20 :   SDValue Mask = MLD->getMask();
    1172          20 :   SDValue Src0 = MLD->getSrc0();
    1173          20 :   unsigned Alignment = MLD->getOriginalAlignment();
    1174             :   ISD::LoadExtType ExtType = MLD->getExtensionType();
    1175             : 
    1176             :   // if Alignment is equal to the vector size,
    1177             :   // take the half of it for the second part
    1178             :   unsigned SecondHalfAlignment =
    1179          40 :     (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
    1180             :      Alignment/2 : Alignment;
    1181             : 
    1182             :   // Split Mask operand
    1183          20 :   SDValue MaskLo, MaskHi;
    1184          60 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1185           1 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1186             :   else
    1187          38 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1188             : 
    1189          20 :   EVT MemoryVT = MLD->getMemoryVT();
    1190          20 :   EVT LoMemVT, HiMemVT;
    1191          40 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1192             : 
    1193          20 :   SDValue Src0Lo, Src0Hi;
    1194          60 :   if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
    1195          20 :     GetSplitVector(Src0, Src0Lo, Src0Hi);
    1196             :   else
    1197           0 :     std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
    1198             : 
    1199          20 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1200          40 :     getMachineMemOperand(MLD->getPointerInfo(),
    1201             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1202          20 :                          Alignment, MLD->getAAInfo(), MLD->getRanges());
    1203             : 
    1204          40 :   Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
    1205          20 :                          ExtType, MLD->isExpandingLoad());
    1206             : 
    1207          40 :   Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
    1208          20 :                                    MLD->isExpandingLoad());
    1209             :   unsigned HiOffset = LoMemVT.getStoreSize();
    1210             : 
    1211          60 :   MMO = DAG.getMachineFunction().getMachineMemOperand(
    1212             :       MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
    1213          20 :       HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
    1214             :       MLD->getRanges());
    1215             : 
    1216          40 :   Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
    1217          20 :                          ExtType, MLD->isExpandingLoad());
    1218             : 
    1219             :   // Build a factor node to remember that this load is independent of the
    1220             :   // other one.
    1221          40 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1222          40 :                    Hi.getValue(1));
    1223             : 
    1224             :   // Legalize the chain result - switch anything that used the old chain to
    1225             :   // use the new one.
    1226          20 :   ReplaceValueWith(SDValue(MLD, 1), Ch);
    1227             : 
    1228          20 : }
    1229             : 
    1230          16 : void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
    1231             :                                          SDValue &Lo, SDValue &Hi) {
    1232          16 :   EVT LoVT, HiVT;
    1233             :   SDLoc dl(MGT);
    1234          48 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
    1235             : 
    1236          16 :   SDValue Ch = MGT->getChain();
    1237          16 :   SDValue Ptr = MGT->getBasePtr();
    1238          16 :   SDValue Mask = MGT->getMask();
    1239          16 :   SDValue Src0 = MGT->getValue();
    1240          16 :   SDValue Index = MGT->getIndex();
    1241          16 :   SDValue Scale = MGT->getScale();
    1242          16 :   unsigned Alignment = MGT->getOriginalAlignment();
    1243             : 
    1244             :   // Split Mask operand
    1245          16 :   SDValue MaskLo, MaskHi;
    1246          48 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1247           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1248             :   else
    1249          32 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1250             : 
    1251          16 :   EVT MemoryVT = MGT->getMemoryVT();
    1252          16 :   EVT LoMemVT, HiMemVT;
    1253             :   // Split MemoryVT
    1254          32 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1255             : 
    1256          16 :   SDValue Src0Lo, Src0Hi;
    1257          48 :   if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
    1258          16 :     GetSplitVector(Src0, Src0Lo, Src0Hi);
    1259             :   else
    1260           0 :     std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
    1261             : 
    1262          16 :   SDValue IndexHi, IndexLo;
    1263          48 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1264          12 :     GetSplitVector(Index, IndexLo, IndexHi);
    1265             :   else
    1266           8 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
    1267             : 
    1268          16 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1269          32 :     getMachineMemOperand(MGT->getPointerInfo(),
    1270             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1271          16 :                          Alignment, MGT->getAAInfo(), MGT->getRanges());
    1272             : 
    1273          16 :   SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
    1274          32 :   Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
    1275          32 :                            MMO);
    1276             : 
    1277          16 :   SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
    1278          32 :   Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
    1279          32 :                            MMO);
    1280             : 
    1281             :   // Build a factor node to remember that this load is independent of the
    1282             :   // other one.
    1283          32 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1284          32 :                    Hi.getValue(1));
    1285             : 
    1286             :   // Legalize the chain result - switch anything that used the old chain to
    1287             :   // use the new one.
    1288          16 :   ReplaceValueWith(SDValue(MGT, 1), Ch);
    1289          16 : }
    1290             : 
    1291             : 
    1292        1838 : void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
    1293             :   assert(N->getValueType(0).isVector() &&
    1294             :          N->getOperand(0).getValueType().isVector() &&
    1295             :          "Operand types must be vectors");
    1296             : 
    1297        1838 :   EVT LoVT, HiVT;
    1298             :   SDLoc DL(N);
    1299        5514 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1300             : 
    1301             :   // If the input also splits, handle it directly. Otherwise split it by hand.
    1302        1838 :   SDValue LL, LH, RL, RH;
    1303        5514 :   if (getTypeAction(N->getOperand(0).getValueType()) ==
    1304             :       TargetLowering::TypeSplitVector)
    1305        3438 :     GetSplitVector(N->getOperand(0), LL, LH);
    1306             :   else
    1307         238 :     std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
    1308             : 
    1309        5514 :   if (getTypeAction(N->getOperand(1).getValueType()) ==
    1310             :       TargetLowering::TypeSplitVector)
    1311        3438 :     GetSplitVector(N->getOperand(1), RL, RH);
    1312             :   else
    1313         238 :     std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
    1314             : 
    1315        7352 :   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
    1316        7352 :   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
    1317        1838 : }
    1318             : 
    1319       10412 : void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
    1320             :                                            SDValue &Hi) {
    1321             :   // Get the dest types - they may not match the input types, e.g. int_to_fp.
    1322       10412 :   EVT LoVT, HiVT;
    1323             :   SDLoc dl(N);
    1324       31236 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    1325             : 
    1326             :   // If the input also splits, handle it directly for a compile time speedup.
    1327             :   // Otherwise split it by hand.
    1328       10412 :   EVT InVT = N->getOperand(0).getValueType();
    1329       20824 :   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
    1330       12754 :     GetSplitVector(N->getOperand(0), Lo, Hi);
    1331             :   else
    1332        8070 :     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
    1333             : 
    1334       20824 :   if (N->getOpcode() == ISD::FP_ROUND) {
    1335        1809 :     Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
    1336        2412 :     Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
    1337             :   } else {
    1338       19618 :     Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
    1339       29427 :     Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
    1340             :   }
    1341       10412 : }
    1342             : 
    1343        7511 : void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
    1344             :                                             SDValue &Hi) {
    1345             :   SDLoc dl(N);
    1346       15022 :   EVT SrcVT = N->getOperand(0).getValueType();
    1347       15022 :   EVT DestVT = N->getValueType(0);
    1348             :   EVT LoVT, HiVT;
    1349       15022 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
    1350             : 
    1351             :   // We can do better than a generic split operation if the extend is doing
    1352             :   // more than just doubling the width of the elements and the following are
    1353             :   // true:
    1354             :   //   - The number of vector elements is even,
    1355             :   //   - the source type is legal,
    1356             :   //   - the type of a split source is illegal,
    1357             :   //   - the type of an extended (by doubling element size) source is legal, and
    1358             :   //   - the type of that extended source when split is legal.
    1359             :   //
    1360             :   // This won't necessarily completely legalize the operation, but it will
    1361             :   // more effectively move in the right direction and prevent falling down
    1362             :   // to scalarization in many cases due to the input vector being split too
    1363             :   // far.
    1364        7511 :   unsigned NumElements = SrcVT.getVectorNumElements();
    1365       15022 :   if ((NumElements & 1) == 0 &&
    1366        7511 :       SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
    1367        5019 :     LLVMContext &Ctx = *DAG.getContext();
    1368        5019 :     EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
    1369        5019 :     EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
    1370             : 
    1371             :     EVT SplitLoVT, SplitHiVT;
    1372       10038 :     std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
    1373        5372 :     if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
    1374         163 :         TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
    1375             :       LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
    1376             :                  N->dump(&DAG); dbgs() << "\n");
    1377             :       // Extend the source vector by one step.
    1378             :       SDValue NewSrc =
    1379         280 :           DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
    1380             :       // Get the low and high halves of the new, extended one step, vector.
    1381         140 :       std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
    1382             :       // Extend those vector halves the rest of the way.
    1383         210 :       Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
    1384         210 :       Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
    1385             :       return;
    1386             :     }
    1387             :   }
    1388             :   // Fall back to the generic unary operator splitting otherwise.
    1389        7441 :   SplitVecRes_UnaryOp(N, Lo, Hi);
    1390             : }
    1391             : 
    1392        4838 : void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
    1393             :                                                   SDValue &Lo, SDValue &Hi) {
    1394             :   // The low and high parts of the original input give four input vectors.
    1395        4838 :   SDValue Inputs[4];
    1396             :   SDLoc dl(N);
    1397        9676 :   GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
    1398        9676 :   GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
    1399        9676 :   EVT NewVT = Inputs[0].getValueType();
    1400        4838 :   unsigned NewElts = NewVT.getVectorNumElements();
    1401             : 
    1402             :   // If Lo or Hi uses elements from at most two of the four input vectors, then
    1403             :   // express it as a vector shuffle of those two inputs.  Otherwise extract the
    1404             :   // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
    1405             :   SmallVector<int, 16> Ops;
    1406       24190 :   for (unsigned High = 0; High < 2; ++High) {
    1407        9676 :     SDValue &Output = High ? Hi : Lo;
    1408             : 
    1409             :     // Build a shuffle mask for the output, discovering on the fly which
    1410             :     // input vectors to use as shuffle operands (recorded in InputUsed).
    1411             :     // If building a suitable shuffle vector proves too hard, then bail
    1412             :     // out with useBuildVector set.
    1413        9676 :     unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
    1414        9676 :     unsigned FirstMaskIdx = High * NewElts;
    1415             :     bool useBuildVector = false;
    1416      285270 :     for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
    1417             :       // The mask element.  This indexes into the input.
    1418      137876 :       int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
    1419             : 
    1420             :       // The input vector this mask element indexes into.
    1421      137876 :       unsigned Input = (unsigned)Idx / NewElts;
    1422             : 
    1423      241640 :       if (Input >= array_lengthof(Inputs)) {
    1424             :         // The mask element does not index into any input vector.
    1425      103764 :         Ops.push_back(-1);
    1426      103764 :         continue;
    1427             :       }
    1428             : 
    1429             :       // Turn the index into an offset from the start of the input vector.
    1430       34112 :       Idx -= Input * NewElts;
    1431             : 
    1432             :       // Find or create a shuffle vector operand to hold this input.
    1433             :       unsigned OpNo;
    1434       44462 :       for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
    1435       39208 :         if (InputUsed[OpNo] == Input) {
    1436             :           // This input vector is already an operand.
    1437             :           break;
    1438       11877 :         } else if (InputUsed[OpNo] == -1U) {
    1439             :           // Create a new operand for this input vector.
    1440        6702 :           InputUsed[OpNo] = Input;
    1441        6702 :           break;
    1442             :         }
    1443             :       }
    1444             : 
    1445       34112 :       if (OpNo >= array_lengthof(InputUsed)) {
    1446             :         // More than two input vectors used!  Give up on trying to create a
    1447             :         // shuffle vector.  Insert all elements into a BUILD_VECTOR instead.
    1448             :         useBuildVector = true;
    1449             :         break;
    1450             :       }
    1451             : 
    1452             :       // Add the mask index for the new shuffle vector.
    1453       34033 :       Ops.push_back(Idx + OpNo * NewElts);
    1454             :     }
    1455             : 
    1456        9676 :     if (useBuildVector) {
    1457          79 :       EVT EltVT = NewVT.getVectorElementType();
    1458             :       SmallVector<SDValue, 16> SVOps;
    1459             : 
    1460             :       // Extract the input elements by hand.
    1461        1303 :       for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
    1462             :         // The mask element.  This indexes into the input.
    1463         612 :         int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
    1464             : 
    1465             :         // The input vector this mask element indexes into.
    1466         612 :         unsigned Input = (unsigned)Idx / NewElts;
    1467             : 
    1468         735 :         if (Input >= array_lengthof(Inputs)) {
    1469             :           // The mask element is "undef" or indexes off the end of the input.
    1470         123 :           SVOps.push_back(DAG.getUNDEF(EltVT));
    1471         123 :           continue;
    1472             :         }
    1473             : 
    1474             :         // Turn the index into an offset from the start of the input vector.
    1475         489 :         Idx -= Input * NewElts;
    1476             : 
    1477             :         // Extract the vector element by hand.
    1478         978 :         SVOps.push_back(DAG.getNode(
    1479             :             ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
    1480        1956 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    1481             :       }
    1482             : 
    1483             :       // Construct the Lo/Hi output using a BUILD_VECTOR.
    1484         158 :       Output = DAG.getBuildVector(NewVT, dl, SVOps);
    1485        9597 :     } else if (InputUsed[0] == -1U) {
    1486             :       // No input vectors were used!  The result is undefined.
    1487        4007 :       Output = DAG.getUNDEF(NewVT);
    1488             :     } else {
    1489        5590 :       SDValue Op0 = Inputs[InputUsed[0]];
    1490             :       // If only one input was used, use an undefined vector for the other.
    1491        5590 :       SDValue Op1 = InputUsed[1] == -1U ?
    1492        5590 :         DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
    1493             :       // At least one input vector was used.  Create a new shuffle vector.
    1494       11180 :       Output =  DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
    1495             :     }
    1496             : 
    1497             :     Ops.clear();
    1498             :   }
    1499        4838 : }
    1500             : 
    1501             : 
    1502             : //===----------------------------------------------------------------------===//
    1503             : //  Operand Vector Splitting
    1504             : //===----------------------------------------------------------------------===//
    1505             : 
    1506             : /// This method is called when the specified operand of the specified node is
    1507             : /// found to need vector splitting. At this point, all of the result types of
    1508             : /// the node are known to be legal, but other operands of the node may need
    1509             : /// legalization as well as the specified one.
    1510       58856 : bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
    1511             :   LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
    1512       58856 :   SDValue Res = SDValue();
    1513             : 
    1514             :   // See if the target wants to custom split this node.
    1515      176568 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    1516             :     return false;
    1517             : 
    1518             :   if (!Res.getNode()) {
    1519      117132 :     switch (N->getOpcode()) {
    1520           0 :     default:
    1521             : #ifndef NDEBUG
    1522             :       dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
    1523             :       N->dump(&DAG);
    1524             :       dbgs() << "\n";
    1525             : #endif
    1526           0 :       report_fatal_error("Do not know how to split this operator's "
    1527             :                          "operand!\n");
    1528             : 
    1529         141 :     case ISD::SETCC:             Res = SplitVecOp_VSETCC(N); break;
    1530        5609 :     case ISD::BITCAST:           Res = SplitVecOp_BITCAST(N); break;
    1531       20045 :     case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
    1532       13418 :     case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
    1533           2 :     case ISD::CONCAT_VECTORS:    Res = SplitVecOp_CONCAT_VECTORS(N); break;
    1534        1840 :     case ISD::TRUNCATE:
    1535        1840 :       Res = SplitVecOp_TruncateHelper(N);
    1536        1840 :       break;
    1537          81 :     case ISD::FP_ROUND:          Res = SplitVecOp_FP_ROUND(N); break;
    1538           8 :     case ISD::FCOPYSIGN:         Res = SplitVecOp_FCOPYSIGN(N); break;
    1539             :     case ISD::STORE:
    1540        7834 :       Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
    1541        7834 :       break;
    1542             :     case ISD::MSTORE:
    1543           6 :       Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
    1544           6 :       break;
    1545             :     case ISD::MSCATTER:
    1546          22 :       Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
    1547          22 :       break;
    1548             :     case ISD::MGATHER:
    1549          32 :       Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
    1550          32 :       break;
    1551          58 :     case ISD::VSELECT:
    1552          58 :       Res = SplitVecOp_VSELECT(N, OpNo);
    1553          58 :       break;
    1554          51 :     case ISD::FP_TO_SINT:
    1555             :     case ISD::FP_TO_UINT:
    1556         153 :       if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
    1557          39 :         Res = SplitVecOp_TruncateHelper(N);
    1558             :       else
    1559          12 :         Res = SplitVecOp_UnaryOp(N);
    1560             :       break;
    1561         227 :     case ISD::SINT_TO_FP:
    1562             :     case ISD::UINT_TO_FP:
    1563         681 :       if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
    1564         121 :         Res = SplitVecOp_TruncateHelper(N);
    1565             :       else
    1566         106 :         Res = SplitVecOp_UnaryOp(N);
    1567             :       break;
    1568        9173 :     case ISD::CTTZ:
    1569             :     case ISD::CTLZ:
    1570             :     case ISD::CTPOP:
    1571             :     case ISD::FP_EXTEND:
    1572             :     case ISD::SIGN_EXTEND:
    1573             :     case ISD::ZERO_EXTEND:
    1574             :     case ISD::ANY_EXTEND:
    1575             :     case ISD::FTRUNC:
    1576             :     case ISD::FCANONICALIZE:
    1577        9173 :       Res = SplitVecOp_UnaryOp(N);
    1578        9173 :       break;
    1579             : 
    1580           6 :     case ISD::ANY_EXTEND_VECTOR_INREG:
    1581             :     case ISD::SIGN_EXTEND_VECTOR_INREG:
    1582             :     case ISD::ZERO_EXTEND_VECTOR_INREG:
    1583           6 :       Res = SplitVecOp_ExtVecInRegOp(N);
    1584           6 :       break;
    1585             : 
    1586          13 :     case ISD::VECREDUCE_FADD:
    1587             :     case ISD::VECREDUCE_FMUL:
    1588             :     case ISD::VECREDUCE_ADD:
    1589             :     case ISD::VECREDUCE_MUL:
    1590             :     case ISD::VECREDUCE_AND:
    1591             :     case ISD::VECREDUCE_OR:
    1592             :     case ISD::VECREDUCE_XOR:
    1593             :     case ISD::VECREDUCE_SMAX:
    1594             :     case ISD::VECREDUCE_SMIN:
    1595             :     case ISD::VECREDUCE_UMAX:
    1596             :     case ISD::VECREDUCE_UMIN:
    1597             :     case ISD::VECREDUCE_FMAX:
    1598             :     case ISD::VECREDUCE_FMIN:
    1599          13 :       Res = SplitVecOp_VECREDUCE(N, OpNo);
    1600          13 :       break;
    1601             :     }
    1602             :   }
    1603             : 
    1604             :   // If the result is null, the sub-method took care of registering results etc.
    1605       58566 :   if (!Res.getNode()) return false;
    1606             : 
    1607             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    1608             :   // core about this.
    1609       58534 :   if (Res.getNode() == N)
    1610             :     return true;
    1611             : 
    1612             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    1613             :          "Invalid operand expansion");
    1614             : 
    1615       45911 :   ReplaceValueWith(SDValue(N, 0), Res);
    1616       45911 :   return false;
    1617             : }
    1618             : 
    1619          62 : SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
    1620             :   // The only possibility for an illegal operand is the mask, since result type
    1621             :   // legalization would have handled this node already otherwise.
    1622             :   assert(OpNo == 0 && "Illegal operand must be mask");
    1623             : 
    1624          62 :   SDValue Mask = N->getOperand(0);
    1625          62 :   SDValue Src0 = N->getOperand(1);
    1626          62 :   SDValue Src1 = N->getOperand(2);
    1627         124 :   EVT Src0VT = Src0.getValueType();
    1628             :   SDLoc DL(N);
    1629             :   assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
    1630             : 
    1631          62 :   SDValue Lo, Hi;
    1632         124 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1633             :   assert(Lo.getValueType() == Hi.getValueType() &&
    1634             :          "Lo and Hi have differing types");
    1635             : 
    1636             :   EVT LoOpVT, HiOpVT;
    1637         124 :   std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
    1638             :   assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
    1639             : 
    1640             :   SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
    1641         124 :   std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
    1642         124 :   std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
    1643         124 :   std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
    1644             : 
    1645             :   SDValue LoSelect =
    1646         124 :     DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
    1647             :   SDValue HiSelect =
    1648         124 :     DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
    1649             : 
    1650         186 :   return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
    1651             : }
    1652             : 
    1653          13 : SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
    1654          26 :   EVT ResVT = N->getValueType(0);
    1655          13 :   SDValue Lo, Hi;
    1656             :   SDLoc dl(N);
    1657             : 
    1658          26 :   SDValue VecOp = N->getOperand(OpNo);
    1659          13 :   EVT VecVT = VecOp.getValueType();
    1660             :   assert(VecVT.isVector() && "Can only split reduce vector operand");
    1661          13 :   GetSplitVector(VecOp, Lo, Hi);
    1662             :   EVT LoOpVT, HiOpVT;
    1663          26 :   std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
    1664             : 
    1665          13 :   bool NoNaN = N->getFlags().hasNoNaNs();
    1666             :   unsigned CombineOpc = 0;
    1667          26 :   switch (N->getOpcode()) {
    1668             :   case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
    1669           0 :   case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
    1670           5 :   case ISD::VECREDUCE_ADD:  CombineOpc = ISD::ADD; break;
    1671           0 :   case ISD::VECREDUCE_MUL:  CombineOpc = ISD::MUL; break;
    1672           0 :   case ISD::VECREDUCE_AND:  CombineOpc = ISD::AND; break;
    1673           0 :   case ISD::VECREDUCE_OR:   CombineOpc = ISD::OR; break;
    1674           0 :   case ISD::VECREDUCE_XOR:  CombineOpc = ISD::XOR; break;
    1675           2 :   case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
    1676           2 :   case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
    1677           2 :   case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
    1678           2 :   case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
    1679           0 :   case ISD::VECREDUCE_FMAX:
    1680           0 :     CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
    1681             :     break;
    1682           0 :   case ISD::VECREDUCE_FMIN:
    1683           0 :     CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
    1684             :     break;
    1685           0 :   default:
    1686           0 :     llvm_unreachable("Unexpected reduce ISD node");
    1687             :   }
    1688             : 
    1689             :   // Use the appropriate scalar instruction on the split subvectors before
    1690             :   // reducing the now partially reduced smaller vector.
    1691          26 :   SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
    1692          52 :   return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
    1693             : }
    1694             : 
    1695       10633 : SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
    1696             :   // The result has a legal vector type, but the input needs splitting.
    1697       21266 :   EVT ResVT = N->getValueType(0);
    1698       10633 :   SDValue Lo, Hi;
    1699             :   SDLoc dl(N);
    1700       21266 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1701       21266 :   EVT InVT = Lo.getValueType();
    1702             : 
    1703       10633 :   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
    1704       21266 :                                InVT.getVectorNumElements());
    1705             : 
    1706       31899 :   Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
    1707       31899 :   Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
    1708             : 
    1709       31899 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
    1710             : }
    1711             : 
    1712        5609 : SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
    1713             :   // For example, i64 = BITCAST v4i16 on alpha.  Typically the vector will
    1714             :   // end up being split all the way down to individual components.  Convert the
    1715             :   // split pieces into integers and reassemble.
    1716        5609 :   SDValue Lo, Hi;
    1717       11218 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1718        5609 :   Lo = BitConvertToInteger(Lo);
    1719        5609 :   Hi = BitConvertToInteger(Hi);
    1720             : 
    1721       11218 :   if (DAG.getDataLayout().isBigEndian())
    1722             :     std::swap(Lo, Hi);
    1723             : 
    1724       11218 :   return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
    1725       16827 :                      JoinIntegers(Lo, Hi));
    1726             : }
    1727             : 
    1728       20045 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
    1729             :   // We know that the extracted result type is legal.
    1730       40090 :   EVT SubVT = N->getValueType(0);
    1731       20045 :   SDValue Idx = N->getOperand(1);
    1732             :   SDLoc dl(N);
    1733       20045 :   SDValue Lo, Hi;
    1734       40090 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    1735             : 
    1736       40090 :   uint64_t LoElts = Lo.getValueType().getVectorNumElements();
    1737       20045 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    1738             : 
    1739       20045 :   if (IdxVal < LoElts) {
    1740             :     assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
    1741             :            "Extracted subvector crosses vector split!");
    1742       20814 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
    1743             :   } else {
    1744        9638 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
    1745             :                        DAG.getConstant(IdxVal - LoElts, dl,
    1746        9638 :                                        Idx.getValueType()));
    1747             :   }
    1748             : }
    1749             : 
    1750       13418 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    1751       13418 :   SDValue Vec = N->getOperand(0);
    1752       13418 :   SDValue Idx = N->getOperand(1);
    1753       13418 :   EVT VecVT = Vec.getValueType();
    1754             : 
    1755             :   if (isa<ConstantSDNode>(Idx)) {
    1756       13200 :     uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    1757             :     assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
    1758             : 
    1759       13200 :     SDValue Lo, Hi;
    1760       13200 :     GetSplitVector(Vec, Lo, Hi);
    1761             : 
    1762       26400 :     uint64_t LoElts = Lo.getValueType().getVectorNumElements();
    1763             : 
    1764       13200 :     if (IdxVal < LoElts)
    1765        7645 :       return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
    1766       11110 :     return SDValue(DAG.UpdateNodeOperands(N, Hi,
    1767        5555 :                                   DAG.getConstant(IdxVal - LoElts, SDLoc(N),
    1768        5555 :                                                   Idx.getValueType())), 0);
    1769             :   }
    1770             : 
    1771             :   // See if the target wants to custom expand this node.
    1772         436 :   if (CustomLowerNode(N, N->getValueType(0), true))
    1773           0 :     return SDValue();
    1774             : 
    1775             :   // Make the vector elements byte-addressable if they aren't already.
    1776             :   SDLoc dl(N);
    1777         218 :   EVT EltVT = VecVT.getVectorElementType();
    1778         218 :   if (VecVT.getScalarSizeInBits() < 8) {
    1779           2 :     EltVT = MVT::i8;
    1780           2 :     VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
    1781           2 :                              VecVT.getVectorNumElements());
    1782           4 :     Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
    1783             :   }
    1784             : 
    1785             :   // Store the vector to the stack.
    1786         218 :   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
    1787         218 :   auto &MF = DAG.getMachineFunction();
    1788         218 :   auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    1789         218 :   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
    1790         436 :   SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
    1791             : 
    1792             :   // Load back the required element.
    1793         218 :   StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
    1794         218 :   return DAG.getExtLoad(
    1795             :       ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
    1796         436 :       MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
    1797             : }
    1798             : 
    1799           6 : SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
    1800           6 :   SDValue Lo, Hi;
    1801             : 
    1802             :   // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
    1803             :   // splitting the result has the same effect as splitting the input operand.
    1804           6 :   SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
    1805             : 
    1806          24 :   return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
    1807             : }
    1808             : 
    1809          32 : SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
    1810             :                                              unsigned OpNo) {
    1811          32 :   EVT LoVT, HiVT;
    1812             :   SDLoc dl(MGT);
    1813          96 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
    1814             : 
    1815          32 :   SDValue Ch = MGT->getChain();
    1816          32 :   SDValue Ptr = MGT->getBasePtr();
    1817          32 :   SDValue Index = MGT->getIndex();
    1818          32 :   SDValue Scale = MGT->getScale();
    1819          32 :   SDValue Mask = MGT->getMask();
    1820          32 :   SDValue Src0 = MGT->getValue();
    1821          32 :   unsigned Alignment = MGT->getOriginalAlignment();
    1822             : 
    1823          32 :   SDValue MaskLo, MaskHi;
    1824          96 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1825             :     // Split Mask operand
    1826           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1827             :   else
    1828          64 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
    1829             : 
    1830          32 :   EVT MemoryVT = MGT->getMemoryVT();
    1831          32 :   EVT LoMemVT, HiMemVT;
    1832          64 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1833             : 
    1834          32 :   SDValue Src0Lo, Src0Hi;
    1835          96 :   if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
    1836           0 :     GetSplitVector(Src0, Src0Lo, Src0Hi);
    1837             :   else
    1838          64 :     std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
    1839             : 
    1840          32 :   SDValue IndexHi, IndexLo;
    1841          96 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1842          32 :     GetSplitVector(Index, IndexLo, IndexHi);
    1843             :   else
    1844           0 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
    1845             : 
    1846          32 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1847          64 :     getMachineMemOperand(MGT->getPointerInfo(),
    1848             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    1849          32 :                          Alignment, MGT->getAAInfo(), MGT->getRanges());
    1850             : 
    1851          32 :   SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
    1852          32 :   SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
    1853          32 :                                    OpsLo, MMO);
    1854             : 
    1855          64 :   MMO = DAG.getMachineFunction().
    1856          64 :     getMachineMemOperand(MGT->getPointerInfo(),
    1857             :                          MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),
    1858          32 :                          Alignment, MGT->getAAInfo(),
    1859             :                          MGT->getRanges());
    1860             : 
    1861          32 :   SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
    1862          32 :   SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
    1863          32 :                                    OpsHi, MMO);
    1864             : 
    1865             :   // Build a factor node to remember that this load is independent of the
    1866             :   // other one.
    1867          64 :   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
    1868          64 :                    Hi.getValue(1));
    1869             : 
    1870             :   // Legalize the chain result - switch anything that used the old chain to
    1871             :   // use the new one.
    1872          32 :   ReplaceValueWith(SDValue(MGT, 1), Ch);
    1873             : 
    1874          32 :   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
    1875          64 :                             Hi);
    1876          32 :   ReplaceValueWith(SDValue(MGT, 0), Res);
    1877          64 :   return SDValue();
    1878             : }
    1879             : 
    1880           8 : SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
    1881             :                                             unsigned OpNo) {
    1882           8 :   SDValue Ch  = N->getChain();
    1883           8 :   SDValue Ptr = N->getBasePtr();
    1884           8 :   SDValue Mask = N->getMask();
    1885           8 :   SDValue Data = N->getValue();
    1886           8 :   EVT MemoryVT = N->getMemoryVT();
    1887           8 :   unsigned Alignment = N->getOriginalAlignment();
    1888             :   SDLoc DL(N);
    1889             : 
    1890           8 :   EVT LoMemVT, HiMemVT;
    1891          16 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1892             : 
    1893           8 :   SDValue DataLo, DataHi;
    1894          24 :   if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
    1895             :     // Split Data operand
    1896           8 :     GetSplitVector(Data, DataLo, DataHi);
    1897             :   else
    1898           0 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    1899             : 
    1900           8 :   SDValue MaskLo, MaskHi;
    1901          24 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1902             :     // Split Mask operand
    1903           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1904             :   else
    1905          16 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
    1906             : 
    1907             :   // if Alignment is equal to the vector size,
    1908             :   // take the half of it for the second part
    1909             :   unsigned SecondHalfAlignment =
    1910          16 :     (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
    1911             :        Alignment/2 : Alignment;
    1912             : 
    1913             :   SDValue Lo, Hi;
    1914           8 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1915          16 :     getMachineMemOperand(N->getPointerInfo(),
    1916             :                          MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
    1917           8 :                          Alignment, N->getAAInfo(), N->getRanges());
    1918             : 
    1919          16 :   Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
    1920             :                           N->isTruncatingStore(),
    1921          16 :                           N->isCompressingStore());
    1922             : 
    1923          16 :   Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    1924           8 :                                    N->isCompressingStore());
    1925             :   unsigned HiOffset = LoMemVT.getStoreSize();
    1926             : 
    1927          24 :   MMO = DAG.getMachineFunction().getMachineMemOperand(
    1928             :       N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
    1929           8 :       HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
    1930             :       N->getRanges());
    1931             : 
    1932          16 :   Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
    1933          16 :                           N->isTruncatingStore(), N->isCompressingStore());
    1934             : 
    1935             :   // Build a factor node to remember that this store is independent of the
    1936             :   // other one.
    1937          24 :   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    1938             : }
    1939             : 
    1940          22 : SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
    1941             :                                               unsigned OpNo) {
    1942          22 :   SDValue Ch  = N->getChain();
    1943          22 :   SDValue Ptr = N->getBasePtr();
    1944          22 :   SDValue Mask = N->getMask();
    1945          22 :   SDValue Index = N->getIndex();
    1946          22 :   SDValue Scale = N->getScale();
    1947          22 :   SDValue Data = N->getValue();
    1948          22 :   EVT MemoryVT = N->getMemoryVT();
    1949          22 :   unsigned Alignment = N->getOriginalAlignment();
    1950             :   SDLoc DL(N);
    1951             : 
    1952             :   // Split all operands
    1953          22 :   EVT LoMemVT, HiMemVT;
    1954          44 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    1955             : 
    1956          22 :   SDValue DataLo, DataHi;
    1957          66 :   if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
    1958             :     // Split Data operand
    1959          13 :     GetSplitVector(Data, DataLo, DataHi);
    1960             :   else
    1961          18 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    1962             : 
    1963          22 :   SDValue MaskLo, MaskHi;
    1964          66 :   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
    1965             :     // Split Mask operand
    1966           0 :     GetSplitVector(Mask, MaskLo, MaskHi);
    1967             :   else
    1968          44 :     std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
    1969             : 
    1970          22 :   SDValue IndexHi, IndexLo;
    1971          66 :   if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
    1972          18 :     GetSplitVector(Index, IndexLo, IndexHi);
    1973             :   else
    1974           8 :     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
    1975             : 
    1976             :   SDValue Lo;
    1977          22 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    1978          44 :     getMachineMemOperand(N->getPointerInfo(),
    1979             :                          MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
    1980          22 :                          Alignment, N->getAAInfo(), N->getRanges());
    1981             : 
    1982          22 :   SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
    1983          44 :   Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
    1984          66 :                             DL, OpsLo, MMO);
    1985             : 
    1986          44 :   MMO = DAG.getMachineFunction().
    1987          44 :     getMachineMemOperand(N->getPointerInfo(),
    1988             :                          MachineMemOperand::MOStore,  HiMemVT.getStoreSize(),
    1989          22 :                          Alignment, N->getAAInfo(), N->getRanges());
    1990             : 
    1991             :   // The order of the Scatter operation after split is well defined. The "Hi"
    1992             :   // part comes after the "Lo". So these two operations should be chained one
    1993             :   // after another.
    1994          22 :   SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
    1995          22 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
    1996          66 :                               DL, OpsHi, MMO);
    1997             : }
    1998             : 
    1999        7834 : SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
    2000             :   assert(N->isUnindexed() && "Indexed store of vector?");
    2001             :   assert(OpNo == 1 && "Can only split the stored value");
    2002             :   SDLoc DL(N);
    2003             : 
    2004             :   bool isTruncating = N->isTruncatingStore();
    2005        7834 :   SDValue Ch  = N->getChain();
    2006        7834 :   SDValue Ptr = N->getBasePtr();
    2007        7834 :   EVT MemoryVT = N->getMemoryVT();
    2008        7834 :   unsigned Alignment = N->getOriginalAlignment();
    2009        7834 :   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
    2010             :   AAMDNodes AAInfo = N->getAAInfo();
    2011        7834 :   SDValue Lo, Hi;
    2012        7834 :   GetSplitVector(N->getOperand(1), Lo, Hi);
    2013             : 
    2014        7834 :   EVT LoMemVT, HiMemVT;
    2015       15668 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    2016             : 
    2017             :   // Scalarize if the split halves are not byte-sized.
    2018       15631 :   if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
    2019          37 :     return TLI.scalarizeVectorStore(N, DAG);
    2020             : 
    2021        7797 :   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
    2022             : 
    2023        7797 :   if (isTruncating)
    2024           0 :     Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
    2025           0 :                            Alignment, MMOFlags, AAInfo);
    2026             :   else
    2027       23391 :     Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
    2028        7797 :                       AAInfo);
    2029             : 
    2030             :   // Increment the pointer to the other half.
    2031        7797 :   Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
    2032             : 
    2033        7797 :   if (isTruncating)
    2034           0 :     Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
    2035           0 :                            N->getPointerInfo().getWithOffset(IncrementSize),
    2036           0 :                            HiMemVT, Alignment, MMOFlags, AAInfo);
    2037             :   else
    2038       15594 :     Hi = DAG.getStore(Ch, DL, Hi, Ptr,
    2039        7797 :                       N->getPointerInfo().getWithOffset(IncrementSize),
    2040       15594 :                       Alignment, MMOFlags, AAInfo);
    2041             : 
    2042       15594 :   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    2043             : }
    2044             : 
    2045           2 : SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
    2046             :   SDLoc DL(N);
    2047             : 
    2048             :   // The input operands all must have the same type, and we know the result
    2049             :   // type is valid.  Convert this to a buildvector which extracts all the
    2050             :   // input elements.
    2051             :   // TODO: If the input elements are power-two vectors, we could convert this to
    2052             :   // a new CONCAT_VECTORS node with elements that are half-wide.
    2053             :   SmallVector<SDValue, 32> Elts;
    2054           4 :   EVT EltVT = N->getValueType(0).getVectorElementType();
    2055          10 :   for (const SDValue &Op : N->op_values()) {
    2056          64 :     for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
    2057          56 :          i != e; ++i) {
    2058          96 :       Elts.push_back(DAG.getNode(
    2059             :           ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
    2060         192 :           DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    2061             :     }
    2062             :   }
    2063             : 
    2064           8 :   return DAG.getBuildVector(N->getValueType(0), DL, Elts);
    2065             : }
    2066             : 
    2067        2000 : SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
    2068             :   // The result type is legal, but the input type is illegal.  If splitting
    2069             :   // ends up with the result type of each half still being legal, just
    2070             :   // do that.  If, however, that would result in an illegal result type,
    2071             :   // we can try to get more clever with power-two vectors. Specifically,
    2072             :   // split the input type, but also widen the result element size, then
    2073             :   // concatenate the halves and truncate again.  For example, consider a target
    2074             :   // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
    2075             :   // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
    2076             :   //   %inlo = v4i32 extract_subvector %in, 0
    2077             :   //   %inhi = v4i32 extract_subvector %in, 4
    2078             :   //   %lo16 = v4i16 trunc v4i32 %inlo
    2079             :   //   %hi16 = v4i16 trunc v4i32 %inhi
    2080             :   //   %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
    2081             :   //   %res = v8i8 trunc v8i16 %in16
    2082             :   //
    2083             :   // Without this transform, the original truncate would end up being
    2084             :   // scalarized, which is pretty much always a last resort.
    2085        2000 :   SDValue InVec = N->getOperand(0);
    2086        4000 :   EVT InVT = InVec->getValueType(0);
    2087        4000 :   EVT OutVT = N->getValueType(0);
    2088        2000 :   unsigned NumElements = OutVT.getVectorNumElements();
    2089        2000 :   bool IsFloat = OutVT.isFloatingPoint();
    2090             : 
    2091             :   // Widening should have already made sure this is a power-two vector
    2092             :   // if we're trying to split it at all. assert() that's true, just in case.
    2093             :   assert(!(NumElements & 1) && "Splitting vector, but not in half!");
    2094             : 
    2095             :   unsigned InElementSize = InVT.getScalarSizeInBits();
    2096             :   unsigned OutElementSize = OutVT.getScalarSizeInBits();
    2097             : 
    2098             :   // If the input elements are only 1/2 the width of the result elements,
    2099             :   // just use the normal splitting. Our trick only work if there's room
    2100             :   // to split more than once.
    2101        2000 :   if (InElementSize <= OutElementSize * 2)
    2102        1342 :     return SplitVecOp_UnaryOp(N);
    2103             :   SDLoc DL(N);
    2104             : 
    2105             :   // Get the split input vector.
    2106         658 :   SDValue InLoVec, InHiVec;
    2107         658 :   GetSplitVector(InVec, InLoVec, InHiVec);
    2108             :   // Truncate them to 1/2 the element size.
    2109             :   EVT HalfElementVT = IsFloat ?
    2110          21 :     EVT::getFloatingPointVT(InElementSize/2) :
    2111         679 :     EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
    2112         658 :   EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
    2113        1316 :                                 NumElements/2);
    2114        1974 :   SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
    2115        1974 :   SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
    2116             :   // Concatenate them to get the full intermediate truncation result.
    2117         658 :   EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
    2118         658 :   SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
    2119         658 :                                  HalfHi);
    2120             :   // Now finish up by truncating all the way down to the original result
    2121             :   // type. This should normally be something that ends up being legal directly,
    2122             :   // but in theory if a target has very wide vectors and an annoyingly
    2123             :   // restricted set of legal types, this split can chain to build things up.
    2124             :   return IsFloat
    2125          21 :              ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
    2126             :                            DAG.getTargetConstant(
    2127         679 :                                0, DL, TLI.getPointerTy(DAG.getDataLayout())))
    2128        1316 :              : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
    2129             : }
    2130             : 
    2131         143 : SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
    2132             :   assert(N->getValueType(0).isVector() &&
    2133             :          N->getOperand(0).getValueType().isVector() &&
    2134             :          "Operand types must be vectors");
    2135             :   // The result has a legal vector type, but the input needs splitting.
    2136         143 :   SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
    2137             :   SDLoc DL(N);
    2138         286 :   GetSplitVector(N->getOperand(0), Lo0, Hi0);
    2139         286 :   GetSplitVector(N->getOperand(1), Lo1, Hi1);
    2140         286 :   unsigned PartElements = Lo0.getValueType().getVectorNumElements();
    2141         143 :   EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
    2142         286 :   EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
    2143             : 
    2144         429 :   LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
    2145         429 :   HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
    2146         286 :   SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
    2147         429 :   return PromoteTargetBoolean(Con, N->getValueType(0));
    2148             : }
    2149             : 
    2150             : 
    2151          81 : SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
    2152             :   // The result has a legal vector type, but the input needs splitting.
    2153         162 :   EVT ResVT = N->getValueType(0);
    2154          81 :   SDValue Lo, Hi;
    2155             :   SDLoc DL(N);
    2156         162 :   GetSplitVector(N->getOperand(0), Lo, Hi);
    2157         162 :   EVT InVT = Lo.getValueType();
    2158             : 
    2159          81 :   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
    2160         162 :                                InVT.getVectorNumElements());
    2161             : 
    2162         243 :   Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
    2163         243 :   Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
    2164             : 
    2165         243 :   return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
    2166             : }
    2167             : 
    2168           8 : SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
    2169             :   // The result (and the first input) has a legal vector type, but the second
    2170             :   // input needs splitting.
    2171          16 :   return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
    2172             : }
    2173             : 
    2174             : 
    2175             : //===----------------------------------------------------------------------===//
    2176             : //  Result Vector Widening
    2177             : //===----------------------------------------------------------------------===//
    2178             : 
    2179        6516 : void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
    2180             :   LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
    2181             :              dbgs() << "\n");
    2182             : 
    2183             :   // See if the target wants to custom widen this node.
    2184       13032 :   if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
    2185         123 :     return;
    2186             : 
    2187        6393 :   SDValue Res = SDValue();
    2188       12786 :   switch (N->getOpcode()) {
    2189           0 :   default:
    2190             : #ifndef NDEBUG
    2191             :     dbgs() << "WidenVectorResult #" << ResNo << ": ";
    2192             :     N->dump(&DAG);
    2193             :     dbgs() << "\n";
    2194             : #endif
    2195           0 :     llvm_unreachable("Do not know how to widen the result of this operator!");
    2196             : 
    2197           0 :   case ISD::MERGE_VALUES:      Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
    2198         687 :   case ISD::BITCAST:           Res = WidenVecRes_BITCAST(N); break;
    2199         503 :   case ISD::BUILD_VECTOR:      Res = WidenVecRes_BUILD_VECTOR(N); break;
    2200          46 :   case ISD::CONCAT_VECTORS:    Res = WidenVecRes_CONCAT_VECTORS(N); break;
    2201        1699 :   case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
    2202           0 :   case ISD::FP_ROUND_INREG:    Res = WidenVecRes_InregOp(N); break;
    2203          40 :   case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
    2204        1099 :   case ISD::LOAD:              Res = WidenVecRes_LOAD(N); break;
    2205           6 :   case ISD::SCALAR_TO_VECTOR:  Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
    2206          13 :   case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
    2207         108 :   case ISD::VSELECT:
    2208         108 :   case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
    2209           0 :   case ISD::SELECT_CC:         Res = WidenVecRes_SELECT_CC(N); break;
    2210         101 :   case ISD::SETCC:             Res = WidenVecRes_SETCC(N); break;
    2211         639 :   case ISD::UNDEF:             Res = WidenVecRes_UNDEF(N); break;
    2212             :   case ISD::VECTOR_SHUFFLE:
    2213         116 :     Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
    2214         116 :     break;
    2215             :   case ISD::MLOAD:
    2216          10 :     Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
    2217          10 :     break;
    2218             :   case ISD::MGATHER:
    2219          18 :     Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
    2220          18 :     break;
    2221             : 
    2222         205 :   case ISD::ADD:
    2223             :   case ISD::AND:
    2224             :   case ISD::MUL:
    2225             :   case ISD::MULHS:
    2226             :   case ISD::MULHU:
    2227             :   case ISD::OR:
    2228             :   case ISD::SUB:
    2229             :   case ISD::XOR:
    2230             :   case ISD::FMINNUM:
    2231             :   case ISD::FMAXNUM:
    2232             :   case ISD::FMINNAN:
    2233             :   case ISD::FMAXNAN:
    2234             :   case ISD::SMIN:
    2235             :   case ISD::SMAX:
    2236             :   case ISD::UMIN:
    2237             :   case ISD::UMAX:
    2238         205 :     Res = WidenVecRes_Binary(N);
    2239         205 :     break;
    2240             : 
    2241         154 :   case ISD::FADD:
    2242             :   case ISD::FMUL:
    2243             :   case ISD::FPOW:
    2244             :   case ISD::FSUB:
    2245             :   case ISD::FDIV:
    2246             :   case ISD::FREM:
    2247             :   case ISD::SDIV:
    2248             :   case ISD::UDIV:
    2249             :   case ISD::SREM:
    2250             :   case ISD::UREM:
    2251         154 :     Res = WidenVecRes_BinaryCanTrap(N);
    2252         154 :     break;
    2253             : 
    2254           5 :   case ISD::FCOPYSIGN:
    2255           5 :     Res = WidenVecRes_FCOPYSIGN(N);
    2256           5 :     break;
    2257             : 
    2258           1 :   case ISD::FPOWI:
    2259           1 :     Res = WidenVecRes_POWI(N);
    2260           1 :     break;
    2261             : 
    2262          26 :   case ISD::SHL:
    2263             :   case ISD::SRA:
    2264             :   case ISD::SRL:
    2265          26 :     Res = WidenVecRes_Shift(N);
    2266          26 :     break;
    2267             : 
    2268          14 :   case ISD::ANY_EXTEND_VECTOR_INREG:
    2269             :   case ISD::SIGN_EXTEND_VECTOR_INREG:
    2270             :   case ISD::ZERO_EXTEND_VECTOR_INREG:
    2271          14 :     Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
    2272          14 :     break;
    2273             : 
    2274         890 :   case ISD::ANY_EXTEND:
    2275             :   case ISD::FP_EXTEND:
    2276             :   case ISD::FP_ROUND:
    2277             :   case ISD::FP_TO_SINT:
    2278             :   case ISD::FP_TO_UINT:
    2279             :   case ISD::SIGN_EXTEND:
    2280             :   case ISD::SINT_TO_FP:
    2281             :   case ISD::TRUNCATE:
    2282             :   case ISD::UINT_TO_FP:
    2283             :   case ISD::ZERO_EXTEND:
    2284         890 :     Res = WidenVecRes_Convert(N);
    2285         890 :     break;
    2286             : 
    2287          11 :   case ISD::BITREVERSE:
    2288             :   case ISD::BSWAP:
    2289             :   case ISD::CTLZ:
    2290             :   case ISD::CTPOP:
    2291             :   case ISD::CTTZ:
    2292             :   case ISD::FABS:
    2293             :   case ISD::FCEIL:
    2294             :   case ISD::FCOS:
    2295             :   case ISD::FEXP:
    2296             :   case ISD::FEXP2:
    2297             :   case ISD::FFLOOR:
    2298             :   case ISD::FLOG:
    2299             :   case ISD::FLOG10:
    2300             :   case ISD::FLOG2:
    2301             :   case ISD::FNEARBYINT:
    2302             :   case ISD::FNEG:
    2303             :   case ISD::FRINT:
    2304             :   case ISD::FROUND:
    2305             :   case ISD::FSIN:
    2306             :   case ISD::FSQRT:
    2307             :   case ISD::FTRUNC:
    2308          11 :     Res = WidenVecRes_Unary(N);
    2309          11 :     break;
    2310           2 :   case ISD::FMA:
    2311           2 :     Res = WidenVecRes_Ternary(N);
    2312           2 :     break;
    2313             :   }
    2314             : 
    2315             :   // If Res is null, the sub-method took care of registering the result.
    2316        6393 :   if (Res.getNode())
    2317        6393 :     SetWidenedVector(SDValue(N, ResNo), Res);
    2318             : }
    2319             : 
    2320           2 : SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
    2321             :   // Ternary op widening.
    2322             :   SDLoc dl(N);
    2323           6 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2324           4 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2325           4 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2326           4 :   SDValue InOp3 = GetWidenedVector(N->getOperand(2));
    2327           8 :   return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
    2328             : }
    2329             : 
    2330         205 : SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
    2331             :   // Binary op widening.
    2332             :   SDLoc dl(N);
    2333         615 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2334         410 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2335         410 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2336         820 :   return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
    2337             : }
    2338             : 
    2339         158 : SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
    2340             :   // Binary op widening for operations that can trap.
    2341         158 :   unsigned Opcode = N->getOpcode();
    2342             :   SDLoc dl(N);
    2343         474 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2344         158 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    2345         158 :   EVT VT = WidenVT;
    2346         158 :   unsigned NumElts =  VT.getVectorNumElements();
    2347         158 :   const SDNodeFlags Flags = N->getFlags();
    2348         243 :   while (!TLI.isTypeLegal(VT) && NumElts != 1) {
    2349          27 :     NumElts = NumElts / 2;
    2350          27 :     VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2351             :   }
    2352             : 
    2353         466 :   if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
    2354             :     // Operation doesn't trap so just widen as normal.
    2355         256 :     SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2356         256 :     SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2357         256 :     return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
    2358             :   }
    2359             : 
    2360             :   // No legal vector version so unroll the vector operation and then widen.
    2361          30 :   if (NumElts == 1)
    2362           4 :     return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
    2363             : 
    2364             :   // Since the operation can trap, apply operation on the original vector.
    2365          26 :   EVT MaxVT = VT;
    2366          52 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    2367          52 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    2368          52 :   unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
    2369             : 
    2370          52 :   SmallVector<SDValue, 16> ConcatOps(CurNumElts);
    2371             :   unsigned ConcatEnd = 0;  // Current ConcatOps index.
    2372             :   int Idx = 0;        // Current Idx into input vectors.
    2373             : 
    2374             :   // NumElts := greatest legal vector size (at most WidenVT)
    2375             :   // while (orig. vector has unhandled elements) {
    2376             :   //   take munches of size NumElts from the beginning and add to ConcatOps
    2377             :   //   NumElts := next smaller supported vector size or 1
    2378             :   // }
    2379          56 :   while (CurNumElts != 0) {
    2380          54 :     while (CurNumElts >= NumElts) {
    2381          12 :       SDValue EOp1 = DAG.getNode(
    2382             :           ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
    2383          36 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2384          12 :       SDValue EOp2 = DAG.getNode(
    2385             :           ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
    2386          36 :           DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2387          24 :       ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
    2388          12 :       Idx += NumElts;
    2389          12 :       CurNumElts -= NumElts;
    2390             :     }
    2391             :     do {
    2392          57 :       NumElts = NumElts / 2;
    2393          57 :       VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
    2394         110 :     } while (!TLI.isTypeLegal(VT) && NumElts != 1);
    2395             : 
    2396          30 :     if (NumElts == 1) {
    2397         108 :       for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
    2398          41 :         SDValue EOp1 = DAG.getNode(
    2399             :             ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
    2400         123 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2401          41 :         SDValue EOp2 = DAG.getNode(
    2402             :             ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
    2403         123 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2404          82 :         ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
    2405             :                                              EOp1, EOp2, Flags);
    2406             :       }
    2407             :       CurNumElts = 0;
    2408             :     }
    2409             :   }
    2410             : 
    2411             :   // Check to see if we have a single operation with the widen type.
    2412          26 :   if (ConcatEnd == 1) {
    2413          24 :     VT = ConcatOps[0].getValueType();
    2414          12 :     if (VT == WidenVT)
    2415           0 :       return ConcatOps[0];
    2416             :   }
    2417             : 
    2418             :   // while (Some element of ConcatOps is not of type MaxVT) {
    2419             :   //   From the end of ConcatOps, collect elements of the same type and put
    2420             :   //   them into an op of the next larger supported type
    2421             :   // }
    2422         198 :   while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
    2423          30 :     Idx = ConcatEnd - 1;
    2424          90 :     VT = ConcatOps[Idx--].getValueType();
    2425          88 :     while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
    2426          15 :       Idx--;
    2427             : 
    2428          30 :     int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
    2429             :     EVT NextVT;
    2430             :     do {
    2431          57 :       NextSize *= 2;
    2432          57 :       NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
    2433          57 :     } while (!TLI.isTypeLegal(NextVT));
    2434             : 
    2435          30 :     if (!VT.isVector()) {
    2436             :       // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
    2437          26 :       SDValue VecOp = DAG.getUNDEF(NextVT);
    2438          26 :       unsigned NumToInsert = ConcatEnd - Idx - 1;
    2439          67 :       for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
    2440          82 :         VecOp = DAG.getNode(
    2441          41 :             ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
    2442         164 :             DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2443             :       }
    2444          52 :       ConcatOps[Idx+1] = VecOp;
    2445          26 :       ConcatEnd = Idx + 2;
    2446             :     } else {
    2447             :       // Vector type, create a CONCAT_VECTORS of type NextVT
    2448           4 :       SDValue undefVec = DAG.getUNDEF(VT);
    2449           4 :       unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
    2450           8 :       SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
    2451           4 :       unsigned RealVals = ConcatEnd - Idx - 1;
    2452             :       unsigned SubConcatEnd = 0;
    2453           4 :       unsigned SubConcatIdx = Idx + 1;
    2454          12 :       while (SubConcatEnd < RealVals)
    2455          12 :         SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
    2456          12 :       while (SubConcatEnd < OpsToConcat)
    2457           8 :         SubConcatOps[SubConcatEnd++] = undefVec;
    2458          12 :       ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
    2459           4 :                                             NextVT, SubConcatOps);
    2460           4 :       ConcatEnd = SubConcatIdx + 1;
    2461             :     }
    2462             :   }
    2463             : 
    2464             :   // Check to see if we have a single operation with the widen type.
    2465          26 :   if (ConcatEnd == 1) {
    2466          34 :     VT = ConcatOps[0].getValueType();
    2467          17 :     if (VT == WidenVT)
    2468          17 :       return ConcatOps[0];
    2469             :   }
    2470             : 
    2471             :   // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
    2472           9 :   unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
    2473           9 :   if (NumOps != ConcatEnd ) {
    2474           3 :     SDValue UndefVal = DAG.getUNDEF(MaxVT);
    2475           9 :     for (unsigned j = ConcatEnd; j < NumOps; ++j)
    2476           6 :       ConcatOps[j] = UndefVal;
    2477             :   }
    2478           9 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
    2479           9 :                      makeArrayRef(ConcatOps.data(), NumOps));
    2480             : }
    2481             : 
    2482         890 : SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
    2483         890 :   SDValue InOp = N->getOperand(0);
    2484             :   SDLoc DL(N);
    2485             : 
    2486        2670 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2487         890 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2488             : 
    2489         890 :   EVT InVT = InOp.getValueType();
    2490         890 :   EVT InEltVT = InVT.getVectorElementType();
    2491         890 :   EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
    2492             : 
    2493         890 :   unsigned Opcode = N->getOpcode();
    2494         890 :   unsigned InVTNumElts = InVT.getVectorNumElements();
    2495         890 :   const SDNodeFlags Flags = N->getFlags();
    2496        1780 :   if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
    2497         506 :     InOp = GetWidenedVector(N->getOperand(0));
    2498         253 :     InVT = InOp.getValueType();
    2499         253 :     InVTNumElts = InVT.getVectorNumElements();
    2500         253 :     if (InVTNumElts == WidenNumElts) {
    2501         240 :       if (N->getNumOperands() == 1)
    2502         456 :         return DAG.getNode(Opcode, DL, WidenVT, InOp);
    2503          24 :       return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
    2504             :     }
    2505          13 :     if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
    2506             :       // If both input and result vector types are of same width, extend
    2507             :       // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
    2508             :       // accepts fewer elements in the result than in the input.
    2509          13 :       if (Opcode == ISD::SIGN_EXTEND)
    2510           4 :         return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
    2511           9 :       if (Opcode == ISD::ZERO_EXTEND)
    2512           5 :         return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
    2513             :     }
    2514             :   }
    2515             : 
    2516         641 :   if (TLI.isTypeLegal(InWidenVT)) {
    2517             :     // Because the result and the input are different vector types, widening
    2518             :     // the result could create a legal type but widening the input might make
    2519             :     // it an illegal type that might lead to repeatedly splitting the input
    2520             :     // and then widening it. To avoid this, we widen the input only if
    2521             :     // it results in a legal type.
    2522          56 :     if (WidenNumElts % InVTNumElts == 0) {
    2523             :       // Widen the input and call convert on the widened input vector.
    2524          56 :       unsigned NumConcat = WidenNumElts/InVTNumElts;
    2525         112 :       SmallVector<SDValue, 16> Ops(NumConcat);
    2526          56 :       Ops[0] = InOp;
    2527          56 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    2528         168 :       for (unsigned i = 1; i != NumConcat; ++i)
    2529         112 :         Ops[i] = UndefVal;
    2530         112 :       SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
    2531          56 :       if (N->getNumOperands() == 1)
    2532         112 :         return DAG.getNode(Opcode, DL, WidenVT, InVec);
    2533           0 :       return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
    2534             :     }
    2535             : 
    2536           0 :     if (InVTNumElts % WidenNumElts == 0) {
    2537           0 :       SDValue InVal = DAG.getNode(
    2538             :           ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
    2539           0 :           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2540             :       // Extract the input and convert the shorten input vector.
    2541           0 :       if (N->getNumOperands() == 1)
    2542           0 :         return DAG.getNode(Opcode, DL, WidenVT, InVal);
    2543           0 :       return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
    2544             :     }
    2545             :   }
    2546             : 
    2547             :   // Otherwise unroll into some nasty scalar code and rebuild the vector.
    2548        1170 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    2549         585 :   EVT EltVT = WidenVT.getVectorElementType();
    2550         585 :   unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
    2551             :   unsigned i;
    2552        3841 :   for (i=0; i < MinElts; ++i) {
    2553        1628 :     SDValue Val = DAG.getNode(
    2554             :         ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
    2555        4884 :         DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2556        1628 :     if (N->getNumOperands() == 1)
    2557        4788 :       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
    2558             :     else
    2559          96 :       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
    2560             :   }
    2561             : 
    2562         585 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    2563        3985 :   for (; i < WidenNumElts; ++i)
    2564        3400 :     Ops[i] = UndefVal;
    2565             : 
    2566        1170 :   return DAG.getBuildVector(WidenVT, DL, Ops);
    2567             : }
    2568             : 
    2569          14 : SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
    2570          14 :   unsigned Opcode = N->getOpcode();
    2571          14 :   SDValue InOp = N->getOperand(0);
    2572             :   SDLoc DL(N);
    2573             : 
    2574          42 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2575          14 :   EVT WidenSVT = WidenVT.getVectorElementType();
    2576          14 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2577             : 
    2578          14 :   EVT InVT = InOp.getValueType();
    2579          14 :   EVT InSVT = InVT.getVectorElementType();
    2580          14 :   unsigned InVTNumElts = InVT.getVectorNumElements();
    2581             : 
    2582          28 :   if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
    2583          14 :     InOp = GetWidenedVector(InOp);
    2584          14 :     InVT = InOp.getValueType();
    2585          14 :     if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
    2586          14 :       switch (Opcode) {
    2587           0 :       case ISD::ANY_EXTEND_VECTOR_INREG:
    2588           0 :         return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
    2589           5 :       case ISD::SIGN_EXTEND_VECTOR_INREG:
    2590           5 :         return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
    2591           9 :       case ISD::ZERO_EXTEND_VECTOR_INREG:
    2592           9 :         return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
    2593             :       }
    2594             :     }
    2595             :   }
    2596             : 
    2597             :   // Unroll, extend the scalars and rebuild the vector.
    2598             :   SmallVector<SDValue, 16> Ops;
    2599           0 :   for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
    2600           0 :     SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
    2601           0 :       DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2602           0 :     switch (Opcode) {
    2603           0 :     case ISD::ANY_EXTEND_VECTOR_INREG:
    2604           0 :       Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
    2605           0 :       break;
    2606           0 :     case ISD::SIGN_EXTEND_VECTOR_INREG:
    2607           0 :       Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
    2608           0 :       break;
    2609           0 :     case ISD::ZERO_EXTEND_VECTOR_INREG:
    2610           0 :       Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
    2611           0 :       break;
    2612           0 :     default:
    2613           0 :       llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
    2614             :     }
    2615           0 :     Ops.push_back(Val);
    2616             :   }
    2617             : 
    2618           0 :   while (Ops.size() != WidenNumElts)
    2619           0 :     Ops.push_back(DAG.getUNDEF(WidenSVT));
    2620             : 
    2621           0 :   return DAG.getBuildVector(WidenVT, DL, Ops);
    2622             : }
    2623             : 
    2624           5 : SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
    2625             :   // If this is an FCOPYSIGN with same input types, we can treat it as a
    2626             :   // normal (can trap) binary op.
    2627          13 :   if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
    2628           4 :     return WidenVecRes_BinaryCanTrap(N);
    2629             : 
    2630             :   // If the types are different, fall back to unrolling.
    2631           3 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2632           1 :   return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
    2633             : }
    2634             : 
    2635           1 : SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
    2636           3 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2637           2 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2638           1 :   SDValue ShOp = N->getOperand(1);
    2639           4 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
    2640             : }
    2641             : 
    2642          26 : SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
    2643          78 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2644          52 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2645          26 :   SDValue ShOp = N->getOperand(1);
    2646             : 
    2647          26 :   EVT ShVT = ShOp.getValueType();
    2648          52 :   if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
    2649          26 :     ShOp = GetWidenedVector(ShOp);
    2650          26 :     ShVT = ShOp.getValueType();
    2651             :   }
    2652          26 :   EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
    2653             :                                    ShVT.getVectorElementType(),
    2654          52 :                                    WidenVT.getVectorNumElements());
    2655          26 :   if (ShVT != ShWidenVT)
    2656           0 :     ShOp = ModifyToType(ShOp, ShWidenVT);
    2657             : 
    2658         104 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
    2659             : }
    2660             : 
    2661          11 : SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
    2662             :   // Unary op widening.
    2663          33 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2664          22 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2665          44 :   return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
    2666             : }
    2667             : 
    2668          13 : SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
    2669          39 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2670          13 :   EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
    2671          39 :                                cast<VTSDNode>(N->getOperand(1))->getVT()
    2672             :                                  .getVectorElementType(),
    2673          39 :                                WidenVT.getVectorNumElements());
    2674          26 :   SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
    2675          26 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
    2676          52 :                      WidenVT, WidenLHS, DAG.getValueType(ExtVT));
    2677             : }
    2678             : 
    2679           0 : SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
    2680           0 :   SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
    2681           0 :   return GetWidenedVector(WidenVec);
    2682             : }
    2683             : 
    2684         687 : SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
    2685         687 :   SDValue InOp = N->getOperand(0);
    2686         687 :   EVT InVT = InOp.getValueType();
    2687         687 :   EVT VT = N->getValueType(0);
    2688        1374 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2689             :   SDLoc dl(N);
    2690             : 
    2691        1374 :   switch (getTypeAction(InVT)) {
    2692             :   case TargetLowering::TypeLegal:
    2693             :     break;
    2694             :   case TargetLowering::TypePromoteInteger:
    2695             :     // If the incoming type is a vector that is being promoted, then
    2696             :     // we know that the elements are arranged differently and that we
    2697             :     // must perform the conversion using a stack slot.
    2698          33 :     if (InVT.isVector())
    2699             :       break;
    2700             : 
    2701             :     // If the InOp is promoted to the same size, convert it.  Otherwise,
    2702             :     // fall out of the switch and widen the promoted input.
    2703          16 :     InOp = GetPromotedInteger(InOp);
    2704          16 :     InVT = InOp.getValueType();
    2705          16 :     if (WidenVT.bitsEq(InVT))
    2706          16 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
    2707             :     break;
    2708             :   case TargetLowering::TypeSoftenFloat:
    2709             :   case TargetLowering::TypePromoteFloat:
    2710             :   case TargetLowering::TypeExpandInteger:
    2711             :   case TargetLowering::TypeExpandFloat:
    2712             :   case TargetLowering::TypeScalarizeVector:
    2713             :   case TargetLowering::TypeSplitVector:
    2714             :     break;
    2715         574 :   case TargetLowering::TypeWidenVector:
    2716             :     // If the InOp is widened to the same size, convert it.  Otherwise, fall
    2717             :     // out of the switch and widen the widened input.
    2718         574 :     InOp = GetWidenedVector(InOp);
    2719         574 :     InVT = InOp.getValueType();
    2720         574 :     if (WidenVT.bitsEq(InVT))
    2721             :       // The input widens to the same size. Convert to the widen value.
    2722        1148 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
    2723             :     break;
    2724             :   }
    2725             : 
    2726         105 :   unsigned WidenSize = WidenVT.getSizeInBits();
    2727         105 :   unsigned InSize = InVT.getSizeInBits();
    2728             :   // x86mmx is not an acceptable vector element type, so don't try.
    2729         105 :   if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
    2730             :     // Determine new input vector type.  The new input vector type will use
    2731             :     // the same element type (if its a vector) or use the input type as a
    2732             :     // vector.  It is the same size as the type to widen to.
    2733             :     EVT NewInVT;
    2734          80 :     unsigned NewNumElts = WidenSize / InSize;
    2735          80 :     if (InVT.isVector()) {
    2736          17 :       EVT InEltVT = InVT.getVectorElementType();
    2737          17 :       NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
    2738          17 :                                  WidenSize / InEltVT.getSizeInBits());
    2739             :     } else {
    2740          63 :       NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
    2741             :     }
    2742             : 
    2743          80 :     if (TLI.isTypeLegal(NewInVT)) {
    2744             :       // Because the result and the input are different vector types, widening
    2745             :       // the result could create a legal type but widening the input might make
    2746             :       // it an illegal type that might lead to repeatedly splitting the input
    2747             :       // and then widening it. To avoid this, we widen the input only if
    2748             :       // it results in a legal type.
    2749         156 :       SmallVector<SDValue, 16> Ops(NewNumElts);
    2750          78 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    2751          78 :       Ops[0] = InOp;
    2752         330 :       for (unsigned i = 1; i < NewNumElts; ++i)
    2753         252 :         Ops[i] = UndefVal;
    2754             : 
    2755          78 :       SDValue NewVec;
    2756          78 :       if (InVT.isVector())
    2757          34 :         NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
    2758             :       else
    2759         122 :         NewVec = DAG.getBuildVector(NewInVT, dl, Ops);
    2760         156 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
    2761             :     }
    2762             :   }
    2763             : 
    2764          27 :   return CreateStackStoreLoad(InOp, WidenVT);
    2765             : }
    2766             : 
    2767         503 : SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
    2768             :   SDLoc dl(N);
    2769             :   // Build a vector with undefined for the new nodes.
    2770        1006 :   EVT VT = N->getValueType(0);
    2771             : 
    2772             :   // Integer BUILD_VECTOR operands may be larger than the node's vector element
    2773             :   // type. The UNDEFs need to have the same type as the existing operands.
    2774        1006 :   EVT EltVT = N->getOperand(0).getValueType();
    2775         503 :   unsigned NumElts = VT.getVectorNumElements();
    2776             : 
    2777        1006 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2778         503 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2779             : 
    2780         503 :   SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
    2781             :   assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
    2782         503 :   NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
    2783             : 
    2784        1509 :   return DAG.getBuildVector(WidenVT, dl, NewOps);
    2785             : }
    2786             : 
    2787          46 : SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
    2788          92 :   EVT InVT = N->getOperand(0).getValueType();
    2789         138 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2790             :   SDLoc dl(N);
    2791          46 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2792          46 :   unsigned NumInElts = InVT.getVectorNumElements();
    2793          46 :   unsigned NumOperands = N->getNumOperands();
    2794             : 
    2795             :   bool InputWidened = false; // Indicates we need to widen the input.
    2796          92 :   if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
    2797          13 :     if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
    2798             :       // Add undef vectors to widen to correct length.
    2799             :       unsigned NumConcat = WidenVT.getVectorNumElements() /
    2800          13 :                            InVT.getVectorNumElements();
    2801          13 :       SDValue UndefVal = DAG.getUNDEF(InVT);
    2802          26 :       SmallVector<SDValue, 16> Ops(NumConcat);
    2803         123 :       for (unsigned i=0; i < NumOperands; ++i)
    2804         110 :         Ops[i] = N->getOperand(i);
    2805          71 :       for (unsigned i = NumOperands; i != NumConcat; ++i)
    2806          58 :         Ops[i] = UndefVal;
    2807          26 :       return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
    2808             :     }
    2809             :   } else {
    2810             :     InputWidened = true;
    2811          33 :     if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
    2812             :       // The inputs and the result are widen to the same value.
    2813             :       unsigned i;
    2814          16 :       for (i=1; i < NumOperands; ++i)
    2815          24 :         if (!N->getOperand(i).isUndef())
    2816             :           break;
    2817             : 
    2818           8 :       if (i == NumOperands)
    2819             :         // Everything but the first operand is an UNDEF so just return the
    2820             :         // widened first operand.
    2821           8 :         return GetWidenedVector(N->getOperand(0));
    2822             : 
    2823           4 :       if (NumOperands == 2) {
    2824             :         // Replace concat of two operands with a shuffle.
    2825           8 :         SmallVector<int, 16> MaskOps(WidenNumElts, -1);
    2826          28 :         for (unsigned i = 0; i < NumInElts; ++i) {
    2827          24 :           MaskOps[i] = i;
    2828          24 :           MaskOps[i + NumInElts] = i + WidenNumElts;
    2829             :         }
    2830           4 :         return DAG.getVectorShuffle(WidenVT, dl,
    2831           4 :                                     GetWidenedVector(N->getOperand(0)),
    2832           4 :                                     GetWidenedVector(N->getOperand(1)),
    2833           8 :                                     MaskOps);
    2834             :       }
    2835             :     }
    2836             :   }
    2837             : 
    2838             :   // Fall back to use extracts and build vector.
    2839          25 :   EVT EltVT = WidenVT.getVectorElementType();
    2840          50 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    2841             :   unsigned Idx = 0;
    2842         157 :   for (unsigned i=0; i < NumOperands; ++i) {
    2843         132 :     SDValue InOp = N->getOperand(i);
    2844          66 :     if (InputWidened)
    2845          66 :       InOp = GetWidenedVector(InOp);
    2846        1222 :     for (unsigned j=0; j < NumInElts; ++j)
    2847        1734 :       Ops[Idx++] = DAG.getNode(
    2848             :           ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    2849        2312 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    2850             :   }
    2851          25 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    2852         437 :   for (; Idx < WidenNumElts; ++Idx)
    2853         412 :     Ops[Idx] = UndefVal;
    2854          50 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    2855             : }
    2856             : 
    2857        1699 : SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
    2858        3398 :   EVT      VT = N->getValueType(0);
    2859        3398 :   EVT      WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    2860        1699 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    2861        1699 :   SDValue  InOp = N->getOperand(0);
    2862        1699 :   SDValue  Idx  = N->getOperand(1);
    2863             :   SDLoc dl(N);
    2864             : 
    2865        3398 :   if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
    2866          34 :     InOp = GetWidenedVector(InOp);
    2867             : 
    2868        1699 :   EVT InVT = InOp.getValueType();
    2869             : 
    2870             :   // Check if we can just return the input vector after widening.
    2871        1699 :   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
    2872        1699 :   if (IdxVal == 0 && InVT == WidenVT)
    2873        1452 :     return InOp;
    2874             : 
    2875             :   // Check if we can extract from the vector.
    2876         247 :   unsigned InNumElts = InVT.getVectorNumElements();
    2877         253 :   if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
    2878          12 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
    2879             : 
    2880             :   // We could try widening the input to the right length but for now, extract
    2881             :   // the original elements, fill the rest with undefs and build a vector.
    2882         482 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    2883         241 :   EVT EltVT = VT.getVectorElementType();
    2884         241 :   unsigned NumElts = VT.getVectorNumElements();
    2885             :   unsigned i;
    2886        1633 :   for (i=0; i < NumElts; ++i)
    2887         696 :     Ops[i] =
    2888        2088 :         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    2889             :                     DAG.getConstant(IdxVal + i, dl,
    2890        2784 :                                     TLI.getVectorIdxTy(DAG.getDataLayout())));
    2891             : 
    2892         241 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    2893        1889 :   for (; i < WidenNumElts; ++i)
    2894        1648 :     Ops[i] = UndefVal;
    2895         482 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    2896             : }
    2897             : 
    2898          40 : SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
    2899          80 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    2900          80 :   return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
    2901             :                      InOp.getValueType(), InOp,
    2902         160 :                      N->getOperand(1), N->getOperand(2));
    2903             : }
    2904             : 
    2905        1099 : SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
    2906             :   LoadSDNode *LD = cast<LoadSDNode>(N);
    2907             :   ISD::LoadExtType ExtType = LD->getExtensionType();
    2908             : 
    2909             :   SDValue Result;
    2910             :   SmallVector<SDValue, 16> LdChain;  // Chain for the series of load
    2911        1099 :   if (ExtType != ISD::NON_EXTLOAD)
    2912           0 :     Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
    2913             :   else
    2914        1099 :     Result = GenWidenVectorLoads(LdChain, LD);
    2915             : 
    2916             :   // If we generate a single load, we can use that for the chain.  Otherwise,
    2917             :   // build a factor node to remember the multiple loads are independent and
    2918             :   // chain to that.
    2919        1099 :   SDValue NewChain;
    2920        1099 :   if (LdChain.size() == 1)
    2921         466 :     NewChain = LdChain[0];
    2922             :   else
    2923        1899 :     NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
    2924             : 
    2925             :   // Modified the chain - switch anything that used the old chain to use
    2926             :   // the new one.
    2927        1099 :   ReplaceValueWith(SDValue(N, 1), NewChain);
    2928             : 
    2929        2198 :   return Result;
    2930             : }
    2931             : 
    2932          10 : SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
    2933             : 
    2934          30 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
    2935          10 :   SDValue Mask = N->getMask();
    2936          10 :   EVT MaskVT = Mask.getValueType();
    2937          10 :   SDValue Src0 = GetWidenedVector(N->getSrc0());
    2938             :   ISD::LoadExtType ExtType = N->getExtensionType();
    2939             :   SDLoc dl(N);
    2940             : 
    2941             :   // The mask should be widened as well
    2942          10 :   EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    2943             :                                     MaskVT.getVectorElementType(),
    2944          20 :                                     WidenVT.getVectorNumElements());
    2945          10 :   Mask = ModifyToType(Mask, WideMaskVT, true);
    2946             : 
    2947          10 :   SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
    2948             :                                   Mask, Src0, N->getMemoryVT(),
    2949             :                                   N->getMemOperand(), ExtType,
    2950          20 :                                         N->isExpandingLoad());
    2951             :   // Legalize the chain result - switch anything that used the old chain to
    2952             :   // use the new one.
    2953          10 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    2954          20 :   return Res;
    2955             : }
    2956             : 
    2957          18 : SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
    2958             : 
    2959          54 :   EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2960          18 :   SDValue Mask = N->getMask();
    2961          18 :   EVT MaskVT = Mask.getValueType();
    2962          18 :   SDValue Src0 = GetWidenedVector(N->getValue());
    2963          18 :   SDValue Scale = N->getScale();
    2964          18 :   unsigned NumElts = WideVT.getVectorNumElements();
    2965             :   SDLoc dl(N);
    2966             : 
    2967             :   // The mask should be widened as well
    2968          18 :   EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    2969             :                                     MaskVT.getVectorElementType(),
    2970          36 :                                     WideVT.getVectorNumElements());
    2971          18 :   Mask = ModifyToType(Mask, WideMaskVT, true);
    2972             : 
    2973             :   // Widen the Index operand
    2974          18 :   SDValue Index = N->getIndex();
    2975          18 :   EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
    2976          36 :                                      Index.getValueType().getScalarType(),
    2977          36 :                                      NumElts);
    2978          18 :   Index = ModifyToType(Index, WideIndexVT);
    2979          18 :   SDValue Ops[] = { N->getChain(), Src0, Mask, N->getBasePtr(), Index, Scale };
    2980          18 :   SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
    2981             :                                     N->getMemoryVT(), dl, Ops,
    2982          36 :                                     N->getMemOperand());
    2983             : 
    2984             :   // Legalize the chain result - switch anything that used the old chain to
    2985             :   // use the new one.
    2986          18 :   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
    2987          36 :   return Res;
    2988             : }
    2989             : 
    2990           6 : SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
    2991          18 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    2992          12 :   return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
    2993          18 :                      WidenVT, N->getOperand(0));
    2994             : }
    2995             : 
    2996             : // Return true if this is a node that could have two SETCCs as operands.
    2997             : static inline bool isLogicalMaskOp(unsigned Opcode) {
    2998         834 :   switch (Opcode) {
    2999             :   case ISD::AND:
    3000             :   case ISD::OR:
    3001             :   case ISD::XOR:
    3002             :     return true;
    3003             :   }
    3004             :   return false;
    3005             : }
    3006             : 
    3007             : // This is used just for the assert in convertMask(). Check that this either
    3008             : // a SETCC or a previously handled SETCC by convertMask().
    3009             : #ifndef NDEBUG
    3010             : static inline bool isSETCCorConvertedSETCC(SDValue N) {
    3011             :   if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
    3012             :     N = N.getOperand(0);
    3013             :   else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
    3014             :     for (unsigned i = 1; i < N->getNumOperands(); ++i)
    3015             :       if (!N->getOperand(i)->isUndef())
    3016             :         return false;
    3017             :     N = N.getOperand(0);
    3018             :   }
    3019             : 
    3020             :   if (N.getOpcode() == ISD::TRUNCATE)
    3021             :     N = N.getOperand(0);
    3022             :   else if (N.getOpcode() == ISD::SIGN_EXTEND)
    3023             :     N = N.getOperand(0);
    3024             : 
    3025             :   if (isLogicalMaskOp(N.getOpcode()))
    3026             :     return isSETCCorConvertedSETCC(N.getOperand(0)) &&
    3027             :            isSETCCorConvertedSETCC(N.getOperand(1));
    3028             : 
    3029             :   return (N.getOpcode() == ISD::SETCC ||
    3030             :           ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
    3031             : }
    3032             : #endif
    3033             : 
    3034             : // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
    3035             : // to ToMaskVT if needed with vector extension or truncation.
    3036         990 : SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
    3037             :                                       EVT ToMaskVT) {
    3038             :   // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
    3039             :   // FIXME: This code seems to be too restrictive, we might consider
    3040             :   // generalizing it or dropping it.
    3041             :   assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
    3042             : 
    3043             :   // Make a new Mask node, with a legal result VT.
    3044             :   SmallVector<SDValue, 4> Ops;
    3045        7780 :   for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
    3046        5800 :     Ops.push_back(InMask->getOperand(i));
    3047        3960 :   SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
    3048             : 
    3049             :   // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
    3050             :   // extend or truncate is needed.
    3051         990 :   LLVMContext &Ctx = *DAG.getContext();
    3052             :   unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
    3053             :   unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
    3054         990 :   if (MaskScalarBits < ToMaskScalBits) {
    3055             :     EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
    3056          59 :                                  MaskVT.getVectorNumElements());
    3057         177 :     Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
    3058         931 :   } else if (MaskScalarBits > ToMaskScalBits) {
    3059             :     EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
    3060          87 :                                    MaskVT.getVectorNumElements());
    3061         261 :     Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
    3062             :   }
    3063             : 
    3064             :   assert(Mask->getValueType(0).getScalarSizeInBits() ==
    3065             :              ToMaskVT.getScalarSizeInBits() &&
    3066             :          "Mask should have the right element size by now.");
    3067             : 
    3068             :   // Adjust Mask to the right number of elements.
    3069        1980 :   unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
    3070         990 :   if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
    3071          56 :     MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
    3072          84 :     SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
    3073          84 :     Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
    3074          28 :                        ZeroIdx);
    3075         962 :   } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
    3076          39 :     unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
    3077          39 :     EVT SubVT = Mask->getValueType(0);
    3078          78 :     SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
    3079          39 :     SubOps[0] = Mask;
    3080         117 :     Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
    3081             :   }
    3082             : 
    3083             :   assert((Mask->getValueType(0) == ToMaskVT) &&
    3084             :          "A mask of ToMaskVT should have been produced by now.");
    3085             : 
    3086        1980 :   return Mask;
    3087             : }
    3088             : 
    3089             : // Get the target mask VT, and widen if needed.
    3090         920 : EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
    3091             :   assert(SetCC->getOpcode() == ISD::SETCC);
    3092         920 :   LLVMContext &Ctx = *DAG.getContext();
    3093        1840 :   EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
    3094        1840 :   if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
    3095          62 :     MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
    3096         920 :   return MaskVT;
    3097             : }
    3098             : 
    3099             : // This method tries to handle VSELECT and its mask by legalizing operands
    3100             : // (which may require widening) and if needed adjusting the mask vector type
    3101             : // to match that of the VSELECT. Without it, many cases end up with
    3102             : // scalarization of the SETCC, with many unnecessary instructions.
    3103        1580 : SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
    3104        1580 :   LLVMContext &Ctx = *DAG.getContext();
    3105        1580 :   SDValue Cond = N->getOperand(0);
    3106             : 
    3107        1580 :   if (N->getOpcode() != ISD::VSELECT)
    3108           0 :     return SDValue();
    3109             : 
    3110        3160 :   if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
    3111         690 :     return SDValue();
    3112             : 
    3113             :   // If this is a splitted VSELECT that was previously already handled, do
    3114             :   // nothing.
    3115        1780 :   EVT CondVT = Cond->getValueType(0);
    3116         890 :   if (CondVT.getScalarSizeInBits() != 1)
    3117           1 :     return SDValue();
    3118             : 
    3119        1778 :   EVT VSelVT = N->getValueType(0);
    3120             :   // Only handle vector types which are a power of 2.
    3121         889 :   if (!isPowerOf2_64(VSelVT.getSizeInBits()))
    3122          18 :     return SDValue();
    3123             : 
    3124             :   // Don't touch if this will be scalarized.
    3125         871 :   EVT FinalVT = VSelVT;
    3126        3227 :   while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
    3127         495 :     FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
    3128             : 
    3129         871 :   if (FinalVT.getVectorNumElements() == 1)
    3130          14 :     return SDValue();
    3131             : 
    3132             :   // If there is support for an i1 vector mask, don't touch.
    3133         857 :   if (Cond.getOpcode() == ISD::SETCC) {
    3134         783 :     EVT SetCCOpVT = Cond->getOperand(0).getValueType();
    3135        3108 :     while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
    3136        1028 :       SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
    3137         783 :     EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
    3138         783 :     if (SetCCResVT.getScalarSizeInBits() == 1)
    3139           3 :       return SDValue();
    3140          74 :   } else if (CondVT.getScalarType() == MVT::i1) {
    3141             :     // If there is support for an i1 vector mask (or only scalar i1 conditions),
    3142             :     // don't touch.
    3143         382 :     while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
    3144         156 :       CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
    3145             : 
    3146         144 :     if (CondVT.getScalarType() == MVT::i1)
    3147           4 :       return SDValue();
    3148             :   }
    3149             : 
    3150             :   // Get the VT and operands for VSELECT, and widen if needed.
    3151         850 :   SDValue VSelOp1 = N->getOperand(1);
    3152         850 :   SDValue VSelOp2 = N->getOperand(2);
    3153        1700 :   if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
    3154         122 :     VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
    3155          61 :     VSelOp1 = GetWidenedVector(VSelOp1);
    3156          61 :     VSelOp2 = GetWidenedVector(VSelOp2);
    3157             :   }
    3158             : 
    3159             :   // The mask of the VSELECT should have integer elements.
    3160         850 :   EVT ToMaskVT = VSelVT;
    3161         850 :   if (!ToMaskVT.getScalarType().isInteger())
    3162         407 :     ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
    3163             : 
    3164         850 :   SDValue Mask;
    3165        1700 :   if (Cond->getOpcode() == ISD::SETCC) {
    3166         780 :     EVT MaskVT = getSETCCWidenedResultTy(Cond);
    3167         780 :     Mask = convertMask(Cond, MaskVT, ToMaskVT);
    3168          70 :   } else if (isLogicalMaskOp(Cond->getOpcode()) &&
    3169         140 :              Cond->getOperand(0).getOpcode() == ISD::SETCC &&
    3170          70 :              Cond->getOperand(1).getOpcode() == ISD::SETCC) {
    3171             :     // Cond is (AND/OR/XOR (SETCC, SETCC))
    3172          70 :     SDValue SETCC0 = Cond->getOperand(0);
    3173          70 :     SDValue SETCC1 = Cond->getOperand(1);
    3174          70 :     EVT VT0 = getSETCCWidenedResultTy(SETCC0);
    3175          70 :     EVT VT1 = getSETCCWidenedResultTy(SETCC1);
    3176             :     unsigned ScalarBits0 = VT0.getScalarSizeInBits();
    3177             :     unsigned ScalarBits1 = VT1.getScalarSizeInBits();
    3178             :     unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
    3179             :     EVT MaskVT;
    3180             :     // If the two SETCCs have different VTs, either extend/truncate one of
    3181             :     // them to the other "towards" ToMaskVT, or truncate one and extend the
    3182             :     // other to ToMaskVT.
    3183          70 :     if (ScalarBits0 != ScalarBits1) {
    3184          40 :       EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
    3185          40 :       EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
    3186          40 :       if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
    3187          18 :         MaskVT = WideVT;
    3188          22 :       else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
    3189          22 :         MaskVT = NarrowVT;
    3190             :       else
    3191           0 :         MaskVT = ToMaskVT;
    3192             :     } else
    3193             :       // If the two SETCCs have the same VT, don't change it.
    3194          30 :       MaskVT = VT0;
    3195             : 
    3196             :     // Make new SETCCs and logical nodes.
    3197          70 :     SETCC0 = convertMask(SETCC0, VT0, MaskVT);
    3198          70 :     SETCC1 = convertMask(SETCC1, VT1, MaskVT);
    3199         280 :     Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
    3200             : 
    3201             :     // Convert the logical op for VSELECT if needed.
    3202          70 :     Mask = convertMask(Cond, MaskVT, ToMaskVT);
    3203             :   } else
    3204           0 :     return SDValue();
    3205             : 
    3206        2550 :   return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
    3207             : }
    3208             : 
    3209         108 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
    3210         324 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3211         108 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3212             : 
    3213         108 :   SDValue Cond1 = N->getOperand(0);
    3214         108 :   EVT CondVT = Cond1.getValueType();
    3215         108 :   if (CondVT.isVector()) {
    3216          97 :     if (SDValue Res = WidenVSELECTAndMask(N))
    3217          61 :       return Res;
    3218             : 
    3219          36 :     EVT CondEltVT = CondVT.getVectorElementType();
    3220          36 :     EVT CondWidenVT =  EVT::getVectorVT(*DAG.getContext(),
    3221          36 :                                         CondEltVT, WidenNumElts);
    3222          72 :     if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
    3223          24 :       Cond1 = GetWidenedVector(Cond1);
    3224             : 
    3225             :     // If we have to split the condition there is no point in widening the
    3226             :     // select. This would result in an cycle of widening the select ->
    3227             :     // widening the condition operand -> splitting the condition operand ->
    3228             :     // splitting the select -> widening the select. Instead split this select
    3229             :     // further and widen the resulting type.
    3230          72 :     if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
    3231           4 :       SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
    3232           4 :       SDValue Res = ModifyToType(SplitSelect, WidenVT);
    3233           4 :       return Res;
    3234             :     }
    3235             : 
    3236          32 :     if (Cond1.getValueType() != CondWidenVT)
    3237           8 :       Cond1 = ModifyToType(Cond1, CondWidenVT);
    3238             :   }
    3239             : 
    3240          86 :   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
    3241          86 :   SDValue InOp2 = GetWidenedVector(N->getOperand(2));
    3242             :   assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
    3243          86 :   return DAG.getNode(N->getOpcode(), SDLoc(N),
    3244         129 :                      WidenVT, Cond1, InOp1, InOp2);
    3245             : }
    3246             : 
    3247           0 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
    3248           0 :   SDValue InOp1 = GetWidenedVector(N->getOperand(2));
    3249           0 :   SDValue InOp2 = GetWidenedVector(N->getOperand(3));
    3250           0 :   return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
    3251             :                      InOp1.getValueType(), N->getOperand(0),
    3252           0 :                      N->getOperand(1), InOp1, InOp2, N->getOperand(4));
    3253             : }
    3254             : 
    3255         639 : SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
    3256        1917 :  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3257         639 :  return DAG.getUNDEF(WidenVT);
    3258             : }
    3259             : 
    3260         116 : SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
    3261         232 :   EVT VT = N->getValueType(0);
    3262             :   SDLoc dl(N);
    3263             : 
    3264         232 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    3265         116 :   unsigned NumElts = VT.getVectorNumElements();
    3266         116 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3267             : 
    3268         232 :   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
    3269         232 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    3270             : 
    3271             :   // Adjust mask based on new input vector length.
    3272             :   SmallVector<int, 16> NewMask;
    3273         842 :   for (unsigned i = 0; i != NumElts; ++i) {
    3274         726 :     int Idx = N->getMaskElt(i);
    3275         363 :     if (Idx < (int)NumElts)
    3276         302 :       NewMask.push_back(Idx);
    3277             :     else
    3278          61 :       NewMask.push_back(Idx - NumElts + WidenNumElts);
    3279             :   }
    3280         918 :   for (unsigned i = NumElts; i != WidenNumElts; ++i)
    3281         401 :     NewMask.push_back(-1);
    3282         348 :   return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
    3283             : }
    3284             : 
    3285         101 : SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
    3286             :   assert(N->getValueType(0).isVector() &&
    3287             :          N->getOperand(0).getValueType().isVector() &&
    3288             :          "Operands must be vectors");
    3289         303 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
    3290         101 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3291             : 
    3292         101 :   SDValue InOp1 = N->getOperand(0);
    3293         101 :   EVT InVT = InOp1.getValueType();
    3294             :   assert(InVT.isVector() && "can not widen non-vector type");
    3295         101 :   EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
    3296         202 :                                    InVT.getVectorElementType(), WidenNumElts);
    3297             : 
    3298             :   // The input and output types often differ here, and it could be that while
    3299             :   // we'd prefer to widen the result type, the input operands have been split.
    3300             :   // In this case, we also need to split the result of this node as well.
    3301         202 :   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
    3302           2 :     SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
    3303           2 :     SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
    3304           2 :     return Res;
    3305             :   }
    3306             : 
    3307          99 :   InOp1 = GetWidenedVector(InOp1);
    3308         198 :   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
    3309             : 
    3310             :   // Assume that the input and output will be widen appropriately.  If not,
    3311             :   // we will have to unroll it at some point.
    3312             :   assert(InOp1.getValueType() == WidenInVT &&
    3313             :          InOp2.getValueType() == WidenInVT &&
    3314             :          "Input not widened to expected type!");
    3315             :   (void)WidenInVT;
    3316         198 :   return DAG.getNode(ISD::SETCC, SDLoc(N),
    3317         297 :                      WidenVT, InOp1, InOp2, N->getOperand(2));
    3318             : }
    3319             : 
    3320             : 
    3321             : //===----------------------------------------------------------------------===//
    3322             : // Widen Vector Operand
    3323             : //===----------------------------------------------------------------------===//
    3324        6141 : bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
    3325             :   LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
    3326             :              dbgs() << "\n");
    3327        6141 :   SDValue Res = SDValue();
    3328             : 
    3329             :   // See if the target wants to custom widen this node.
    3330       18423 :   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
    3331             :     return false;
    3332             : 
    3333       12096 :   switch (N->getOpcode()) {
    3334           0 :   default:
    3335             : #ifndef NDEBUG
    3336             :     dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
    3337             :     N->dump(&DAG);
    3338             :     dbgs() << "\n";
    3339             : #endif
    3340           0 :     llvm_unreachable("Do not know how to widen this operator's operand!");
    3341             : 
    3342          12 :   case ISD::BITCAST:            Res = WidenVecOp_BITCAST(N); break;
    3343        1026 :   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
    3344          40 :   case ISD::EXTRACT_SUBVECTOR:  Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
    3345        3749 :   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
    3346         806 :   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
    3347           4 :   case ISD::MSTORE:             Res = WidenVecOp_MSTORE(N, OpNo); break;
    3348           3 :   case ISD::MSCATTER:           Res = WidenVecOp_MSCATTER(N, OpNo); break;
    3349          74 :   case ISD::SETCC:              Res = WidenVecOp_SETCC(N); break;
    3350           3 :   case ISD::FCOPYSIGN:          Res = WidenVecOp_FCOPYSIGN(N); break;
    3351             : 
    3352         215 :   case ISD::ANY_EXTEND:
    3353             :   case ISD::SIGN_EXTEND:
    3354             :   case ISD::ZERO_EXTEND:
    3355         215 :     Res = WidenVecOp_EXTEND(N);
    3356         215 :     break;
    3357             : 
    3358         116 :   case ISD::FP_EXTEND:
    3359             :   case ISD::FP_TO_SINT:
    3360             :   case ISD::FP_TO_UINT:
    3361             :   case ISD::SINT_TO_FP:
    3362             :   case ISD::UINT_TO_FP:
    3363             :   case ISD::TRUNCATE:
    3364         116 :     Res = WidenVecOp_Convert(N);
    3365         116 :     break;
    3366             :   }
    3367             : 
    3368             :   // If Res is null, the sub-method took care of registering the result.
    3369        6048 :   if (!Res.getNode()) return false;
    3370             : 
    3371             :   // If the result is N, the sub-method updated N in place.  Tell the legalizer
    3372             :   // core about this.
    3373        6048 :   if (Res.getNode() == N)
    3374             :     return true;
    3375             : 
    3376             : 
    3377             :   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
    3378             :          "Invalid operand expansion");
    3379             : 
    3380        6048 :   ReplaceValueWith(SDValue(N, 0), Res);
    3381        6048 :   return false;
    3382             : }
    3383             : 
    3384         215 : SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
    3385             :   SDLoc DL(N);
    3386         430 :   EVT VT = N->getValueType(0);
    3387             : 
    3388         215 :   SDValue InOp = N->getOperand(0);
    3389             :   assert(getTypeAction(InOp.getValueType()) ==
    3390             :              TargetLowering::TypeWidenVector &&
    3391             :          "Unexpected type action");
    3392         215 :   InOp = GetWidenedVector(InOp);
    3393             :   assert(VT.getVectorNumElements() <
    3394             :              InOp.getValueType().getVectorNumElements() &&
    3395             :          "Input wasn't widened!");
    3396             : 
    3397             :   // We may need to further widen the operand until it has the same total
    3398             :   // vector size as the result.
    3399         215 :   EVT InVT = InOp.getValueType();
    3400         215 :   if (InVT.getSizeInBits() != VT.getSizeInBits()) {
    3401           0 :     EVT InEltVT = InVT.getVectorElementType();
    3402           0 :     for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
    3403           0 :       EVT FixedVT = (MVT::SimpleValueType)i;
    3404           0 :       EVT FixedEltVT = FixedVT.getVectorElementType();
    3405           0 :       if (TLI.isTypeLegal(FixedVT) &&
    3406           0 :           FixedVT.getSizeInBits() == VT.getSizeInBits() &&
    3407           0 :           FixedEltVT == InEltVT) {
    3408             :         assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
    3409             :                "Not enough elements in the fixed type for the operand!");
    3410             :         assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
    3411             :                "We can't have the same type as we started with!");
    3412           0 :         if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
    3413           0 :           InOp = DAG.getNode(
    3414           0 :               ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
    3415           0 :               DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3416             :         else
    3417           0 :           InOp = DAG.getNode(
    3418             :               ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
    3419           0 :               DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3420           0 :         break;
    3421             :       }
    3422             :     }
    3423           0 :     InVT = InOp.getValueType();
    3424           0 :     if (InVT.getSizeInBits() != VT.getSizeInBits())
    3425             :       // We couldn't find a legal vector type that was a widening of the input
    3426             :       // and could be extended in-register to the result type, so we have to
    3427             :       // scalarize.
    3428           0 :       return WidenVecOp_Convert(N);
    3429             :   }
    3430             : 
    3431             :   // Use special DAG nodes to represent the operation of extending the
    3432             :   // low lanes.
    3433         430 :   switch (N->getOpcode()) {
    3434           0 :   default:
    3435           0 :     llvm_unreachable("Extend legalization on extend operation!");
    3436           2 :   case ISD::ANY_EXTEND:
    3437           2 :     return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
    3438         197 :   case ISD::SIGN_EXTEND:
    3439         197 :     return DAG.getSignExtendVectorInReg(InOp, DL, VT);
    3440          16 :   case ISD::ZERO_EXTEND:
    3441          16 :     return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
    3442             :   }
    3443             : }
    3444             : 
    3445           0 : SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
    3446             :   // The result (and first input) is legal, but the second input is illegal.
    3447             :   // We can't do much to fix that, so just unroll and let the extracts off of
    3448             :   // the second input be widened as needed later.
    3449           3 :   return DAG.UnrollVectorOp(N);
    3450             : }
    3451             : 
    3452         116 : SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
    3453             :   // Since the result is legal and the input is illegal.
    3454         232 :   EVT VT = N->getValueType(0);
    3455         116 :   EVT EltVT = VT.getVectorElementType();
    3456             :   SDLoc dl(N);
    3457         116 :   unsigned NumElts = VT.getVectorNumElements();
    3458         116 :   SDValue InOp = N->getOperand(0);
    3459             :   assert(getTypeAction(InOp.getValueType()) ==
    3460             :              TargetLowering::TypeWidenVector &&
    3461             :          "Unexpected type action");
    3462         116 :   InOp = GetWidenedVector(InOp);
    3463         116 :   EVT InVT = InOp.getValueType();
    3464         116 :   unsigned Opcode = N->getOpcode();
    3465             : 
    3466             :   // See if a widened result type would be legal, if so widen the node.
    3467         116 :   EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
    3468         232 :                                 InVT.getVectorNumElements());
    3469         116 :   if (TLI.isTypeLegal(WideVT)) {
    3470          62 :     SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp);
    3471          31 :     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
    3472          31 :                        DAG.getIntPtrConstant(0, dl));
    3473             :   }
    3474             : 
    3475          85 :   EVT InEltVT = InVT.getVectorElementType();
    3476             : 
    3477             :   // Unroll the convert into some scalar code and create a nasty build vector.
    3478         170 :   SmallVector<SDValue, 16> Ops(NumElts);
    3479         433 :   for (unsigned i=0; i < NumElts; ++i)
    3480         522 :     Ops[i] = DAG.getNode(
    3481             :         Opcode, dl, EltVT,
    3482             :         DAG.getNode(
    3483             :             ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
    3484         696 :             DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
    3485             : 
    3486         170 :   return DAG.getBuildVector(VT, dl, Ops);
    3487             : }
    3488             : 
    3489          12 : SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
    3490          24 :   EVT VT = N->getValueType(0);
    3491          24 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3492          24 :   EVT InWidenVT = InOp.getValueType();
    3493             :   SDLoc dl(N);
    3494             : 
    3495             :   // Check if we can convert between two legal vector types and extract.
    3496          12 :   unsigned InWidenSize = InWidenVT.getSizeInBits();
    3497          12 :   unsigned Size = VT.getSizeInBits();
    3498             :   // x86mmx is not an acceptable vector element type, so don't try.
    3499          24 :   if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
    3500          12 :     unsigned NewNumElts = InWidenSize / Size;
    3501          12 :     EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
    3502          12 :     if (TLI.isTypeLegal(NewVT)) {
    3503          24 :       SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
    3504          12 :       return DAG.getNode(
    3505             :           ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
    3506          36 :           DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3507             :     }
    3508             :   }
    3509             : 
    3510           0 :   return CreateStackStoreLoad(InOp, VT);
    3511             : }
    3512             : 
    3513        1026 : SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
    3514             :   // If the input vector is not legal, it is likely that we will not find a
    3515             :   // legal vector of the same size. Replace the concatenate vector with a
    3516             :   // nasty build vector.
    3517        2052 :   EVT VT = N->getValueType(0);
    3518        1026 :   EVT EltVT = VT.getVectorElementType();
    3519             :   SDLoc dl(N);
    3520        1026 :   unsigned NumElts = VT.getVectorNumElements();
    3521        2052 :   SmallVector<SDValue, 16> Ops(NumElts);
    3522             : 
    3523        2052 :   EVT InVT = N->getOperand(0).getValueType();
    3524        1026 :   unsigned NumInElts = InVT.getVectorNumElements();
    3525             : 
    3526             :   unsigned Idx = 0;
    3527        1026 :   unsigned NumOperands = N->getNumOperands();
    3528        5842 :   for (unsigned i=0; i < NumOperands; ++i) {
    3529        4816 :     SDValue InOp = N->getOperand(i);
    3530             :     assert(getTypeAction(InOp.getValueType()) ==
    3531             :                TargetLowering::TypeWidenVector &&
    3532             :            "Unexpected type action");
    3533        2408 :     InOp = GetWidenedVector(InOp);
    3534       13672 :     for (unsigned j=0; j < NumInElts; ++j)
    3535       16896 :       Ops[Idx++] = DAG.getNode(
    3536             :           ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    3537       22528 :           DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3538             :   }
    3539        3078 :   return DAG.getBuildVector(VT, dl, Ops);
    3540             : }
    3541             : 
    3542          40 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
    3543          80 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3544          80 :   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
    3545         160 :                      N->getValueType(0), InOp, N->getOperand(1));
    3546             : }
    3547             : 
    3548        3749 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
    3549        7498 :   SDValue InOp = GetWidenedVector(N->getOperand(0));
    3550        7498 :   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
    3551       14996 :                      N->getValueType(0), InOp, N->getOperand(1));
    3552             : }
    3553             : 
    3554         806 : SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
    3555             :   // We have to widen the value, but we want only to store the original
    3556             :   // vector type.
    3557             :   StoreSDNode *ST = cast<StoreSDNode>(N);
    3558             : 
    3559        1612 :   if (!ST->getMemoryVT().getScalarType().isByteSized())
    3560          12 :     return TLI.scalarizeVectorStore(ST, DAG);
    3561             : 
    3562             :   SmallVector<SDValue, 16> StChain;
    3563         794 :   if (ST->isTruncatingStore())
    3564           0 :     GenWidenVectorTruncStores(StChain, ST);
    3565             :   else
    3566         794 :     GenWidenVectorStores(StChain, ST);
    3567             : 
    3568         794 :   if (StChain.size() == 1)
    3569         132 :     return StChain[0];
    3570             :   else
    3571        1986 :     return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
    3572             : }
    3573             : 
    3574           6 : SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
    3575             :   assert(OpNo == 3 && "Can widen only data operand of mstore");
    3576             :   MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
    3577           6 :   SDValue Mask = MST->getMask();
    3578           6 :   EVT MaskVT = Mask.getValueType();
    3579           6 :   SDValue StVal = MST->getValue();
    3580             :   // Widen the value
    3581           6 :   SDValue WideVal = GetWidenedVector(StVal);
    3582             :   SDLoc dl(N);
    3583             : 
    3584             :   // The mask should be widened as well.
    3585          12 :   EVT WideVT = WideVal.getValueType();
    3586           6 :   EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    3587             :                                     MaskVT.getVectorElementType(),
    3588          12 :                                     WideVT.getVectorNumElements());
    3589           6 :   Mask = ModifyToType(Mask, WideMaskVT, true);
    3590             : 
    3591             :   assert(Mask.getValueType().getVectorNumElements() ==
    3592             :          WideVal.getValueType().getVectorNumElements() &&
    3593             :          "Mask and data vectors should have the same number of elements");
    3594           6 :   return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
    3595             :                             Mask, MST->getMemoryVT(), MST->getMemOperand(),
    3596          18 :                             false, MST->isCompressingStore());
    3597             : }
    3598             : 
    3599           3 : SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
    3600             :   assert(OpNo == 1 && "Can widen only data operand of mscatter");
    3601             :   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
    3602           3 :   SDValue DataOp = MSC->getValue();
    3603           3 :   SDValue Mask = MSC->getMask();
    3604           3 :   EVT MaskVT = Mask.getValueType();
    3605           3 :   SDValue Scale = MSC->getScale();
    3606             : 
    3607             :   // Widen the value.
    3608           3 :   SDValue WideVal = GetWidenedVector(DataOp);
    3609           6 :   EVT WideVT = WideVal.getValueType();
    3610           3 :   unsigned NumElts = WideVT.getVectorNumElements();
    3611             :   SDLoc dl(N);
    3612             : 
    3613             :   // The mask should be widened as well.
    3614           3 :   EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
    3615           6 :                                     MaskVT.getVectorElementType(), NumElts);
    3616           3 :   Mask = ModifyToType(Mask, WideMaskVT, true);
    3617             : 
    3618             :   // Widen index.
    3619           3 :   SDValue Index = MSC->getIndex();
    3620           3 :   EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
    3621           6 :                                      Index.getValueType().getScalarType(),
    3622           6 :                                      NumElts);
    3623           3 :   Index = ModifyToType(Index, WideIndexVT);
    3624             : 
    3625             :   SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index,
    3626           3 :                    Scale};
    3627           3 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
    3628             :                               MSC->getMemoryVT(), dl, Ops,
    3629           9 :                               MSC->getMemOperand());
    3630             : }
    3631             : 
    3632          74 : SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
    3633         148 :   SDValue InOp0 = GetWidenedVector(N->getOperand(0));
    3634         148 :   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
    3635             :   SDLoc dl(N);
    3636         148 :   EVT VT = N->getValueType(0);
    3637             : 
    3638             :   // WARNING: In this code we widen the compare instruction with garbage.
    3639             :   // This garbage may contain denormal floats which may be slow. Is this a real
    3640             :   // concern ? Should we zero the unused lanes if this is a float compare ?
    3641             : 
    3642             :   // Get a new SETCC node to compare the newly widened operands.
    3643             :   // Only some of the compared elements are legal.
    3644         148 :   EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
    3645         222 :                                    InOp0.getValueType());
    3646             :   // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
    3647         145 :   if (VT.getScalarType() == MVT::i1)
    3648           3 :     SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
    3649           6 :                            SVT.getVectorNumElements());
    3650             : 
    3651         148 :   SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
    3652         222 :                                   SVT, InOp0, InOp1, N->getOperand(2));
    3653             : 
    3654             :   // Extract the needed results from the result vector.
    3655          74 :   EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
    3656             :                                SVT.getVectorElementType(),
    3657         148 :                                VT.getVectorNumElements());
    3658          74 :   SDValue CC = DAG.getNode(
    3659             :       ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
    3660         222 :       DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3661             : 
    3662         148 :   return PromoteTargetBoolean(CC, VT);
    3663             : }
    3664             : 
    3665             : 
    3666             : //===----------------------------------------------------------------------===//
    3667             : // Vector Widening Utilities
    3668             : //===----------------------------------------------------------------------===//
    3669             : 
    3670             : // Utility function to find the type to chop up a widen vector for load/store
    3671             : //  TLI:       Target lowering used to determine legal types.
    3672             : //  Width:     Width left need to load/store.
    3673             : //  WidenVT:   The widen vector type to load to/store from
    3674             : //  Align:     If 0, don't allow use of a wider type
    3675             : //  WidenEx:   If Align is not 0, the amount additional we can load/store from.
    3676             : 
    3677        3040 : static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
    3678             :                        unsigned Width, EVT WidenVT,
    3679             :                        unsigned Align = 0, unsigned WidenEx = 0) {
    3680        3040 :   EVT WidenEltVT = WidenVT.getVectorElementType();
    3681        3040 :   unsigned WidenWidth = WidenVT.getSizeInBits();
    3682        3040 :   unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
    3683        3040 :   unsigned AlignInBits = Align*8;
    3684             : 
    3685             :   // If we have one element to load/store, return it.
    3686        3040 :   EVT RetVT = WidenEltVT;
    3687        3040 :   if (Width == WidenEltWidth)
    3688         919 :     return RetVT;
    3689             : 
    3690             :   // See if there is larger legal integer than the element type to load/store.
    3691             :   unsigned VT;
    3692        2844 :   for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
    3693        4965 :        VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
    3694        4965 :     EVT MemVT((MVT::SimpleValueType) VT);
    3695        4965 :     unsigned MemVTWidth = MemVT.getSizeInBits();
    3696        4965 :     if (MemVT.getSizeInBits() <= WidenEltWidth)
    3697             :       break;
    3698        4623 :     auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
    3699             :     if ((Action == TargetLowering::TypeLegal ||
    3700        2222 :          Action == TargetLowering::TypePromoteInteger) &&
    3701        2222 :         (WidenWidth % MemVTWidth) == 0 &&
    3702        8591 :         isPowerOf2_32(WidenWidth / MemVTWidth) &&
    3703         343 :         (MemVTWidth <= Width ||
    3704         481 :          (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
    3705        1779 :       RetVT = MemVT;
    3706        1779 :       break;
    3707             :     }
    3708             :   }
    3709             : 
    3710             :   // See if there is a larger vector type to load/store that has the same vector
    3711             :   // element type and is evenly divisible with the WidenVT.
    3712      181985 :   for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
    3713      184106 :        VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
    3714      182605 :     EVT MemVT = (MVT::SimpleValueType) VT;
    3715      182605 :     unsigned MemVTWidth = MemVT.getSizeInBits();
    3716       24809 :     if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
    3717        4442 :         (WidenWidth % MemVTWidth) == 0 &&
    3718        5550 :         isPowerOf2_32(WidenWidth / MemVTWidth) &&
    3719        1682 :         (MemVTWidth <= Width ||
    3720        1879 :          (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
    3721        1290 :       if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
    3722         620 :         return MemVT;
    3723             :     }
    3724             :   }
    3725             : 
    3726        1501 :   return RetVT;
    3727             : }
    3728             : 
    3729             : // Builds a vector type from scalar loads
    3730             : //  VecTy: Resulting Vector type
    3731             : //  LDOps: Load operators to build a vector type
    3732             : //  [Start,End) the list of loads to use.
    3733         573 : static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
    3734             :                                      SmallVectorImpl<SDValue> &LdOps,
    3735             :                                      unsigned Start, unsigned End) {
    3736         573 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    3737         573 :   SDLoc dl(LdOps[Start]);
    3738        1146 :   EVT LdTy = LdOps[Start].getValueType();
    3739         573 :   unsigned Width = VecTy.getSizeInBits();
    3740         573 :   unsigned NumElts = Width / LdTy.getSizeInBits();
    3741         573 :   EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
    3742             : 
    3743             :   unsigned Idx = 1;
    3744         573 :   SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
    3745             : 
    3746        1199 :   for (unsigned i = Start + 1; i != End; ++i) {
    3747        1878 :     EVT NewLdTy = LdOps[i].getValueType();
    3748         652 :     if (NewLdTy != LdTy) {
    3749         449 :       NumElts = Width / NewLdTy.getSizeInBits();
    3750         449 :       NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
    3751         449 :       VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
    3752             :       // Readjust position and vector position based on new load type.
    3753         449 :       Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
    3754         449 :       LdTy = NewLdTy;
    3755             :     }
    3756         626 :     VecOp = DAG.getNode(
    3757             :         ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
    3758        2504 :         DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3759             :   }
    3760        1146 :   return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
    3761             : }
    3762             : 
    3763        1099 : SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
    3764             :                                               LoadSDNode *LD) {
    3765             :   // The strategy assumes that we can efficiently load power-of-two widths.
    3766             :   // The routine chops the vector into the largest vector loads with the same
    3767             :   // element type or scalar loads and then recombines it to the widen vector
    3768             :   // type.
    3769        3297 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
    3770        1099 :   unsigned WidenWidth = WidenVT.getSizeInBits();
    3771        1099 :   EVT LdVT    = LD->getMemoryVT();
    3772             :   SDLoc dl(LD);
    3773             :   assert(LdVT.isVector() && WidenVT.isVector());
    3774             :   assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
    3775             : 
    3776             :   // Load information
    3777        1099 :   SDValue Chain = LD->getChain();
    3778        1099 :   SDValue BasePtr = LD->getBasePtr();
    3779        1099 :   unsigned Align = LD->getAlignment();
    3780        1099 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    3781             :   AAMDNodes AAInfo = LD->getAAInfo();
    3782             : 
    3783        1099 :   int LdWidth = LdVT.getSizeInBits();
    3784        1099 :   int WidthDiff = WidenWidth - LdWidth;
    3785        1099 :   unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
    3786             : 
    3787             :   // Find the vector type that can load from.
    3788        1099 :   EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
    3789        1099 :   int NewVTWidth = NewVT.getSizeInBits();
    3790        2198 :   SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
    3791        2198 :                              Align, MMOFlags, AAInfo);
    3792        1099 :   LdChain.push_back(LdOp.getValue(1));
    3793             : 
    3794             :   // Check if we can load the element with one instruction.
    3795        1099 :   if (LdWidth <= NewVTWidth) {
    3796         466 :     if (!NewVT.isVector()) {
    3797         302 :       unsigned NumElts = WidenWidth / NewVTWidth;
    3798         302 :       EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
    3799         604 :       SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
    3800         604 :       return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
    3801             :     }
    3802         164 :     if (NewVT == WidenVT)
    3803         164 :       return LdOp;
    3804             : 
    3805             :     assert(WidenWidth % NewVTWidth == 0);
    3806           0 :     unsigned NumConcat = WidenWidth / NewVTWidth;
    3807           0 :     SmallVector<SDValue, 16> ConcatOps(NumConcat);
    3808           0 :     SDValue UndefVal = DAG.getUNDEF(NewVT);
    3809           0 :     ConcatOps[0] = LdOp;
    3810           0 :     for (unsigned i = 1; i != NumConcat; ++i)
    3811           0 :       ConcatOps[i] = UndefVal;
    3812           0 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
    3813             :   }
    3814             : 
    3815             :   // Load vector by using multiple loads from largest vector to scalar.
    3816             :   SmallVector<SDValue, 16> LdOps;
    3817         633 :   LdOps.push_back(LdOp);
    3818             : 
    3819         633 :   LdWidth -= NewVTWidth;
    3820             :   unsigned Offset = 0;
    3821             : 
    3822        2919 :   while (LdWidth > 0) {
    3823        1143 :     unsigned Increment = NewVTWidth / 8;
    3824        1143 :     Offset += Increment;
    3825        1143 :     BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
    3826             : 
    3827        1143 :     SDValue L;
    3828        1143 :     if (LdWidth < NewVTWidth) {
    3829             :       // The current type we are using is too large. Find a better size.
    3830         538 :       NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
    3831         538 :       NewVTWidth = NewVT.getSizeInBits();
    3832        1076 :       L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
    3833         538 :                       LD->getPointerInfo().getWithOffset(Offset),
    3834        1614 :                       MinAlign(Align, Increment), MMOFlags, AAInfo);
    3835         538 :       LdChain.push_back(L.getValue(1));
    3836        1614 :       if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
    3837             :         // Later code assumes the vector loads produced will be mergeable, so we
    3838             :         // must pad the final entry up to the previous width. Scalars are
    3839             :         // combined separately.
    3840             :         SmallVector<SDValue, 16> Loads;
    3841          41 :         Loads.push_back(L);
    3842          41 :         unsigned size = L->getValueSizeInBits(0);
    3843         109 :         while (size < LdOp->getValueSizeInBits(0)) {
    3844          18 :           Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
    3845          18 :           size += L->getValueSizeInBits(0);
    3846             :         }
    3847          82 :         L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
    3848             :       }
    3849             :     } else {
    3850        1210 :       L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
    3851         605 :                       LD->getPointerInfo().getWithOffset(Offset),
    3852        1815 :                       MinAlign(Align, Increment), MMOFlags, AAInfo);
    3853         605 :       LdChain.push_back(L.getValue(1));
    3854             :     }
    3855             : 
    3856        1143 :     LdOps.push_back(L);
    3857        1143 :     LdOp = L;
    3858             : 
    3859        1143 :     LdWidth -= NewVTWidth;
    3860             :   }
    3861             : 
    3862             :   // Build the vector from the load operations.
    3863         633 :   unsigned End = LdOps.size();
    3864        1899 :   if (!LdOps[0].getValueType().isVector())
    3865             :     // All the loads are scalar loads.
    3866         532 :     return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
    3867             : 
    3868             :   // If the load contains vectors, build the vector using concat vector.
    3869             :   // All of the vectors used to load are power-of-2, and the scalar loads can be
    3870             :   // combined to make a power-of-2 vector.
    3871         202 :   SmallVector<SDValue, 16> ConcatOps(End);
    3872         101 :   int i = End - 1;
    3873         101 :   int Idx = End;
    3874         303 :   EVT LdTy = LdOps[i].getValueType();
    3875             :   // First, combine the scalar loads to a vector.
    3876         101 :   if (!LdTy.isVector())  {
    3877          41 :     for (--i; i >= 0; --i) {
    3878         123 :       LdTy = LdOps[i].getValueType();
    3879          41 :       if (LdTy.isVector())
    3880             :         break;
    3881             :     }
    3882          82 :     ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
    3883             :   }
    3884         303 :   ConcatOps[--Idx] = LdOps[i];
    3885         577 :   for (--i; i >= 0; --i) {
    3886        1428 :     EVT NewLdTy = LdOps[i].getValueType();
    3887         476 :     if (NewLdTy != LdTy) {
    3888             :       // Create a larger vector.
    3889          21 :       ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
    3890          14 :                                      makeArrayRef(&ConcatOps[Idx], End - Idx));
    3891             :       Idx = End - 1;
    3892           7 :       LdTy = NewLdTy;
    3893             :     }
    3894         952 :     ConcatOps[--Idx] = LdOps[i];
    3895             :   }
    3896             : 
    3897         101 :   if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
    3898          67 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
    3899         134 :                        makeArrayRef(&ConcatOps[Idx], End - Idx));
    3900             : 
    3901             :   // We need to fill the rest with undefs to build the vector.
    3902          34 :   unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
    3903          68 :   SmallVector<SDValue, 16> WidenOps(NumOps);
    3904          34 :   SDValue UndefVal = DAG.getUNDEF(LdTy);
    3905             :   {
    3906             :     unsigned i = 0;
    3907         972 :     for (; i != End-Idx; ++i)
    3908        1407 :       WidenOps[i] = ConcatOps[Idx+i];
    3909         424 :     for (; i != NumOps; ++i)
    3910         390 :       WidenOps[i] = UndefVal;
    3911             :   }
    3912          68 :   return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
    3913             : }
    3914             : 
    3915             : SDValue
    3916           0 : DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
    3917             :                                          LoadSDNode *LD,
    3918             :                                          ISD::LoadExtType ExtType) {
    3919             :   // For extension loads, it may not be more efficient to chop up the vector
    3920             :   // and then extend it. Instead, we unroll the load and build a new vector.
    3921           0 :   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
    3922           0 :   EVT LdVT    = LD->getMemoryVT();
    3923             :   SDLoc dl(LD);
    3924             :   assert(LdVT.isVector() && WidenVT.isVector());
    3925             : 
    3926             :   // Load information
    3927           0 :   SDValue Chain = LD->getChain();
    3928           0 :   SDValue BasePtr = LD->getBasePtr();
    3929           0 :   unsigned Align = LD->getAlignment();
    3930           0 :   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
    3931             :   AAMDNodes AAInfo = LD->getAAInfo();
    3932             : 
    3933           0 :   EVT EltVT = WidenVT.getVectorElementType();
    3934           0 :   EVT LdEltVT = LdVT.getVectorElementType();
    3935           0 :   unsigned NumElts = LdVT.getVectorNumElements();
    3936             : 
    3937             :   // Load each element and widen.
    3938           0 :   unsigned WidenNumElts = WidenVT.getVectorNumElements();
    3939           0 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    3940           0 :   unsigned Increment = LdEltVT.getSizeInBits() / 8;
    3941           0 :   Ops[0] =
    3942           0 :       DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
    3943           0 :                      LdEltVT, Align, MMOFlags, AAInfo);
    3944           0 :   LdChain.push_back(Ops[0].getValue(1));
    3945             :   unsigned i = 0, Offset = Increment;
    3946           0 :   for (i=1; i < NumElts; ++i, Offset += Increment) {
    3947           0 :     SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
    3948           0 :     Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
    3949           0 :                             LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
    3950           0 :                             Align, MMOFlags, AAInfo);
    3951           0 :     LdChain.push_back(Ops[i].getValue(1));
    3952             :   }
    3953             : 
    3954             :   // Fill the rest with undefs.
    3955           0 :   SDValue UndefVal = DAG.getUNDEF(EltVT);
    3956           0 :   for (; i != WidenNumElts; ++i)
    3957           0 :     Ops[i] = UndefVal;
    3958             : 
    3959           0 :   return DAG.getBuildVector(WidenVT, dl, Ops);
    3960             : }
    3961             : 
    3962         794 : void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
    3963             :                                             StoreSDNode *ST) {
    3964             :   // The strategy assumes that we can efficiently store power-of-two widths.
    3965             :   // The routine chops the vector into the largest vector stores with the same
    3966             :   // element type or scalar stores.
    3967         794 :   SDValue  Chain = ST->getChain();
    3968         794 :   SDValue  BasePtr = ST->getBasePtr();
    3969         794 :   unsigned Align = ST->getAlignment();
    3970         794 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
    3971             :   AAMDNodes AAInfo = ST->getAAInfo();
    3972         794 :   SDValue  ValOp = GetWidenedVector(ST->getValue());
    3973             :   SDLoc dl(ST);
    3974             : 
    3975         794 :   EVT StVT = ST->getMemoryVT();
    3976         794 :   unsigned StWidth = StVT.getSizeInBits();
    3977        1588 :   EVT ValVT = ValOp.getValueType();
    3978         794 :   unsigned ValWidth = ValVT.getSizeInBits();
    3979         794 :   EVT ValEltVT = ValVT.getVectorElementType();
    3980         794 :   unsigned ValEltWidth = ValEltVT.getSizeInBits();
    3981             :   assert(StVT.getVectorElementType() == ValEltVT);
    3982             : 
    3983             :   int Idx = 0;          // current index to store
    3984             :   unsigned Offset = 0;  // offset from base to store
    3985        3600 :   while (StWidth != 0) {
    3986             :     // Find the largest vector type we can store with.
    3987        1403 :     EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
    3988        1403 :     unsigned NewVTWidth = NewVT.getSizeInBits();
    3989        1403 :     unsigned Increment = NewVTWidth / 8;
    3990        1403 :     if (NewVT.isVector()) {
    3991         307 :       unsigned NumVTElts = NewVT.getVectorNumElements();
    3992             :       do {
    3993         989 :         SDValue EOp = DAG.getNode(
    3994             :             ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
    3995        2967 :             DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    3996        1978 :         StChain.push_back(DAG.getStore(
    3997         989 :             Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
    3998        2967 :             MinAlign(Align, Offset), MMOFlags, AAInfo));
    3999         989 :         StWidth -= NewVTWidth;
    4000         989 :         Offset += Increment;
    4001         989 :         Idx += NumVTElts;
    4002             : 
    4003         989 :         BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
    4004         989 :       } while (StWidth != 0 && StWidth >= NewVTWidth);
    4005             :     } else {
    4006             :       // Cast the vector to the scalar type we can store.
    4007        1096 :       unsigned NumElts = ValWidth / NewVTWidth;
    4008        1096 :       EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
    4009        2192 :       SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
    4010             :       // Readjust index position based on new vector type.
    4011        1096 :       Idx = Idx * ValEltWidth / NewVTWidth;
    4012             :       do {
    4013        1270 :         SDValue EOp = DAG.getNode(
    4014             :             ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
    4015        1270 :             DAG.getConstant(Idx++, dl,
    4016        3810 :                             TLI.getVectorIdxTy(DAG.getDataLayout())));
    4017        2540 :         StChain.push_back(DAG.getStore(
    4018        1270 :             Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
    4019        3810 :             MinAlign(Align, Offset), MMOFlags, AAInfo));
    4020        1270 :         StWidth -= NewVTWidth;
    4021        1270 :         Offset += Increment;
    4022        1270 :         BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
    4023        1270 :       } while (StWidth != 0 && StWidth >= NewVTWidth);
    4024             :       // Restore index back to be relative to the original widen element type.
    4025        1096 :       Idx = Idx * NewVTWidth / ValEltWidth;
    4026             :     }
    4027             :   }
    4028         794 : }
    4029             : 
    4030             : void
    4031           0 : DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
    4032             :                                             StoreSDNode *ST) {
    4033             :   // For extension loads, it may not be more efficient to truncate the vector
    4034             :   // and then store it. Instead, we extract each element and then store it.
    4035           0 :   SDValue Chain = ST->getChain();
    4036           0 :   SDValue BasePtr = ST->getBasePtr();
    4037           0 :   unsigned Align = ST->getAlignment();
    4038           0 :   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
    4039             :   AAMDNodes AAInfo = ST->getAAInfo();
    4040           0 :   SDValue ValOp = GetWidenedVector(ST->getValue());
    4041             :   SDLoc dl(ST);
    4042             : 
    4043           0 :   EVT StVT = ST->getMemoryVT();
    4044           0 :   EVT ValVT = ValOp.getValueType();
    4045             : 
    4046             :   // It must be true that the wide vector type is bigger than where we need to
    4047             :   // store.
    4048             :   assert(StVT.isVector() && ValOp.getValueType().isVector());
    4049             :   assert(StVT.bitsLT(ValOp.getValueType()));
    4050             : 
    4051             :   // For truncating stores, we can not play the tricks of chopping legal vector
    4052             :   // types and bitcast it to the right type. Instead, we unroll the store.
    4053           0 :   EVT StEltVT  = StVT.getVectorElementType();
    4054           0 :   EVT ValEltVT = ValVT.getVectorElementType();
    4055           0 :   unsigned Increment = ValEltVT.getSizeInBits() / 8;
    4056           0 :   unsigned NumElts = StVT.getVectorNumElements();
    4057           0 :   SDValue EOp = DAG.getNode(
    4058             :       ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
    4059           0 :       DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4060           0 :   StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
    4061           0 :                                       ST->getPointerInfo(), StEltVT, Align,
    4062           0 :                                       MMOFlags, AAInfo));
    4063             :   unsigned Offset = Increment;
    4064           0 :   for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
    4065           0 :     SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
    4066           0 :     SDValue EOp = DAG.getNode(
    4067             :         ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
    4068           0 :         DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4069           0 :     StChain.push_back(DAG.getTruncStore(
    4070           0 :         Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
    4071           0 :         StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
    4072             :   }
    4073           0 : }
    4074             : 
    4075             : /// Modifies a vector input (widen or narrows) to a vector of NVT.  The
    4076             : /// input vector must have the same element type as NVT.
    4077             : /// FillWithZeroes specifies that the vector should be widened with zeroes.
    4078          72 : SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
    4079             :                                        bool FillWithZeroes) {
    4080             :   // Note that InOp might have been widened so it might already have
    4081             :   // the right width or it might need be narrowed.
    4082          72 :   EVT InVT = InOp.getValueType();
    4083             :   assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
    4084             :          "input and widen element type must match");
    4085             :   SDLoc dl(InOp);
    4086             : 
    4087             :   // Check if InOp already has the right width.
    4088          72 :   if (InVT == NVT)
    4089           0 :     return InOp;
    4090             : 
    4091          72 :   unsigned InNumElts = InVT.getVectorNumElements();
    4092          72 :   unsigned WidenNumElts = NVT.getVectorNumElements();
    4093          72 :   if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
    4094          72 :     unsigned NumConcat = WidenNumElts / InNumElts;
    4095         144 :     SmallVector<SDValue, 16> Ops(NumConcat);
    4096          37 :     SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
    4097         109 :       DAG.getUNDEF(InVT);
    4098          72 :     Ops[0] = InOp;
    4099         216 :     for (unsigned i = 1; i != NumConcat; ++i)
    4100         144 :       Ops[i] = FillVal;
    4101             : 
    4102         144 :     return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
    4103             :   }
    4104             : 
    4105           0 :   if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
    4106           0 :     return DAG.getNode(
    4107             :         ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
    4108           0 :         DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4109             : 
    4110             :   // Fall back to extract and build.
    4111           0 :   SmallVector<SDValue, 16> Ops(WidenNumElts);
    4112           0 :   EVT EltVT = NVT.getVectorElementType();
    4113           0 :   unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
    4114             :   unsigned Idx;
    4115           0 :   for (Idx = 0; Idx < MinNumElts; ++Idx)
    4116           0 :     Ops[Idx] = DAG.getNode(
    4117             :         ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
    4118           0 :         DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
    4119             : 
    4120           0 :   SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
    4121           0 :     DAG.getUNDEF(EltVT);
    4122           0 :   for ( ; Idx < WidenNumElts; ++Idx)
    4123           0 :     Ops[Idx] = FillVal;
    4124           0 :   return DAG.getBuildVector(NVT, dl, Ops);
    4125             : }

Generated by: LCOV version 1.13