LLVM  7.0.0svn
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16 // types.
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "legalize-types"
30 
31 //===----------------------------------------------------------------------===//
32 // Result Vector Scalarization: <1 x ty> -> ty.
33 //===----------------------------------------------------------------------===//
34 
35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36  DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
37  N->dump(&DAG);
38  dbgs() << "\n");
39  SDValue R = SDValue();
40 
41  switch (N->getOpcode()) {
42  default:
43 #ifndef NDEBUG
44  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45  N->dump(&DAG);
46  dbgs() << "\n";
47 #endif
48  report_fatal_error("Do not know how to scalarize the result of this "
49  "operator!\n");
50 
51  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
57  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
58  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
59  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
60  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
61  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
62  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
63  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
64  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
65  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
66  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
67  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71  R = ScalarizeVecRes_VecInregOp(N);
72  break;
73  case ISD::ANY_EXTEND:
74  case ISD::BITREVERSE:
75  case ISD::BSWAP:
76  case ISD::CTLZ:
78  case ISD::CTPOP:
79  case ISD::CTTZ:
81  case ISD::FABS:
82  case ISD::FCEIL:
83  case ISD::FCOS:
84  case ISD::FEXP:
85  case ISD::FEXP2:
86  case ISD::FFLOOR:
87  case ISD::FLOG:
88  case ISD::FLOG10:
89  case ISD::FLOG2:
90  case ISD::FNEARBYINT:
91  case ISD::FNEG:
92  case ISD::FP_EXTEND:
93  case ISD::FP_TO_SINT:
94  case ISD::FP_TO_UINT:
95  case ISD::FRINT:
96  case ISD::FROUND:
97  case ISD::FSIN:
98  case ISD::FSQRT:
99  case ISD::FTRUNC:
100  case ISD::SIGN_EXTEND:
101  case ISD::SINT_TO_FP:
102  case ISD::TRUNCATE:
103  case ISD::UINT_TO_FP:
104  case ISD::ZERO_EXTEND:
105  case ISD::FCANONICALIZE:
106  R = ScalarizeVecRes_UnaryOp(N);
107  break;
108 
109  case ISD::ADD:
110  case ISD::AND:
111  case ISD::FADD:
112  case ISD::FCOPYSIGN:
113  case ISD::FDIV:
114  case ISD::FMUL:
115  case ISD::FMINNUM:
116  case ISD::FMAXNUM:
117  case ISD::FMINNAN:
118  case ISD::FMAXNAN:
119  case ISD::SMIN:
120  case ISD::SMAX:
121  case ISD::UMIN:
122  case ISD::UMAX:
123 
124  case ISD::FPOW:
125  case ISD::FREM:
126  case ISD::FSUB:
127  case ISD::MUL:
128  case ISD::OR:
129  case ISD::SDIV:
130  case ISD::SREM:
131  case ISD::SUB:
132  case ISD::UDIV:
133  case ISD::UREM:
134  case ISD::XOR:
135  case ISD::SHL:
136  case ISD::SRA:
137  case ISD::SRL:
138  R = ScalarizeVecRes_BinOp(N);
139  break;
140  case ISD::FMA:
141  R = ScalarizeVecRes_TernaryOp(N);
142  break;
143  }
144 
145  // If R is null, the sub-method took care of registering the result.
146  if (R.getNode())
147  SetScalarizedVector(SDValue(N, ResNo), R);
148 }
149 
150 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
151  SDValue LHS = GetScalarizedVector(N->getOperand(0));
152  SDValue RHS = GetScalarizedVector(N->getOperand(1));
153  return DAG.getNode(N->getOpcode(), SDLoc(N),
154  LHS.getValueType(), LHS, RHS, N->getFlags());
155 }
156 
157 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
158  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
159  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
160  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
161  return DAG.getNode(N->getOpcode(), SDLoc(N),
162  Op0.getValueType(), Op0, Op1, Op2);
163 }
164 
165 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
166  unsigned ResNo) {
167  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
168  return GetScalarizedVector(Op);
169 }
170 
171 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
172  SDValue Op = N->getOperand(0);
173  if (Op.getValueType().isVector()
174  && Op.getValueType().getVectorNumElements() == 1
175  && !isSimpleLegalType(Op.getValueType()))
176  Op = GetScalarizedVector(Op);
177  EVT NewVT = N->getValueType(0).getVectorElementType();
178  return DAG.getNode(ISD::BITCAST, SDLoc(N),
179  NewVT, Op);
180 }
181 
182 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
183  EVT EltVT = N->getValueType(0).getVectorElementType();
184  SDValue InOp = N->getOperand(0);
185  // The BUILD_VECTOR operands may be of wider element types and
186  // we may need to truncate them back to the requested return type.
187  if (EltVT.isInteger())
188  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
189  return InOp;
190 }
191 
192 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
193  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
195  N->getOperand(0), N->getOperand(1));
196 }
197 
198 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
199  EVT NewVT = N->getValueType(0).getVectorElementType();
200  SDValue Op = GetScalarizedVector(N->getOperand(0));
201  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
202  NewVT, Op, N->getOperand(1));
203 }
204 
205 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
206  SDValue Op = GetScalarizedVector(N->getOperand(0));
207  return DAG.getNode(ISD::FPOWI, SDLoc(N),
208  Op.getValueType(), Op, N->getOperand(1));
209 }
210 
211 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
212  // The value to insert may have a wider type than the vector element type,
213  // so be sure to truncate it to the element type if necessary.
214  SDValue Op = N->getOperand(1);
215  EVT EltVT = N->getValueType(0).getVectorElementType();
216  if (Op.getValueType() != EltVT)
217  // FIXME: Can this happen for floating point types?
218  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
219  return Op;
220 }
221 
222 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
223  assert(N->isUnindexed() && "Indexed vector load?");
224 
225  SDValue Result = DAG.getLoad(
228  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
231  N->getAAInfo());
232 
233  // Legalize the chain result - switch anything that used the old chain to
234  // use the new one.
235  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
236  return Result;
237 }
238 
239 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
240  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
241  EVT DestVT = N->getValueType(0).getVectorElementType();
242  SDValue Op = N->getOperand(0);
243  EVT OpVT = Op.getValueType();
244  SDLoc DL(N);
245  // The result needs scalarizing, but it's not a given that the source does.
246  // This is a workaround for targets where it's impossible to scalarize the
247  // result of a conversion, because the source type is legal.
248  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
249  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
250  // legal and was not scalarized.
251  // See the similar logic in ScalarizeVecRes_SETCC
252  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
253  Op = GetScalarizedVector(Op);
254  } else {
255  EVT VT = OpVT.getVectorElementType();
256  Op = DAG.getNode(
257  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
258  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
259  }
260  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
261 }
262 
263 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
264  EVT EltVT = N->getValueType(0).getVectorElementType();
265  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
266  SDValue LHS = GetScalarizedVector(N->getOperand(0));
267  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
268  LHS, DAG.getValueType(ExtVT));
269 }
270 
271 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
272  SDLoc DL(N);
273  SDValue Op = N->getOperand(0);
274 
275  EVT OpVT = Op.getValueType();
276  EVT OpEltVT = OpVT.getVectorElementType();
277  EVT EltVT = N->getValueType(0).getVectorElementType();
278 
279  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
280  Op = GetScalarizedVector(Op);
281  } else {
282  Op = DAG.getNode(
283  ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
284  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
285  }
286 
287  switch (N->getOpcode()) {
289  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
291  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
293  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
294  }
295 
296  llvm_unreachable("Illegal extend_vector_inreg opcode");
297 }
298 
299 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
300  // If the operand is wider than the vector element type then it is implicitly
301  // truncated. Make that explicit here.
302  EVT EltVT = N->getValueType(0).getVectorElementType();
303  SDValue InOp = N->getOperand(0);
304  if (InOp.getValueType() != EltVT)
305  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
306  return InOp;
307 }
308 
309 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
310  SDValue Cond = N->getOperand(0);
311  EVT OpVT = Cond.getValueType();
312  SDLoc DL(N);
313  // The vselect result and true/value operands needs scalarizing, but it's
314  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
315  // See the similar logic in ScalarizeVecRes_SETCC
316  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
317  Cond = GetScalarizedVector(Cond);
318  } else {
319  EVT VT = OpVT.getVectorElementType();
320  Cond = DAG.getNode(
321  ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
322  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
323  }
324 
325  SDValue LHS = GetScalarizedVector(N->getOperand(1));
326  TargetLowering::BooleanContent ScalarBool =
327  TLI.getBooleanContents(false, false);
328  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
329 
330  // If integer and float booleans have different contents then we can't
331  // reliably optimize in all cases. There is a full explanation for this in
332  // DAGCombiner::visitSELECT() where the same issue affects folding
333  // (select C, 0, 1) to (xor C, 1).
334  if (TLI.getBooleanContents(false, false) !=
335  TLI.getBooleanContents(false, true)) {
336  // At least try the common case where the boolean is generated by a
337  // comparison.
338  if (Cond->getOpcode() == ISD::SETCC) {
339  EVT OpVT = Cond->getOperand(0).getValueType();
340  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
341  VecBool = TLI.getBooleanContents(OpVT);
342  } else
344  }
345 
346  if (ScalarBool != VecBool) {
347  EVT CondVT = Cond.getValueType();
348  switch (ScalarBool) {
350  break;
354  // Vector read from all ones, scalar expects a single 1 so mask.
355  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
356  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
357  break;
361  // Vector reads from a one, scalar from all ones so sign extend.
362  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
363  Cond, DAG.getValueType(MVT::i1));
364  break;
365  }
366  }
367 
368  return DAG.getSelect(SDLoc(N),
369  LHS.getValueType(), Cond, LHS,
370  GetScalarizedVector(N->getOperand(2)));
371 }
372 
373 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
374  SDValue LHS = GetScalarizedVector(N->getOperand(1));
375  return DAG.getSelect(SDLoc(N),
376  LHS.getValueType(), N->getOperand(0), LHS,
377  GetScalarizedVector(N->getOperand(2)));
378 }
379 
380 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
381  SDValue LHS = GetScalarizedVector(N->getOperand(2));
382  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
383  N->getOperand(0), N->getOperand(1),
384  LHS, GetScalarizedVector(N->getOperand(3)),
385  N->getOperand(4));
386 }
387 
388 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
389  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
390 }
391 
392 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
393  // Figure out if the scalar is the LHS or RHS and return it.
394  SDValue Arg = N->getOperand(2).getOperand(0);
395  if (Arg.isUndef())
396  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
397  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
398  return GetScalarizedVector(N->getOperand(Op));
399 }
400 
401 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
402  assert(N->getValueType(0).isVector() &&
403  N->getOperand(0).getValueType().isVector() &&
404  "Operand types must be vectors");
405  SDValue LHS = N->getOperand(0);
406  SDValue RHS = N->getOperand(1);
407  EVT OpVT = LHS.getValueType();
408  EVT NVT = N->getValueType(0).getVectorElementType();
409  SDLoc DL(N);
410 
411  // The result needs scalarizing, but it's not a given that the source does.
412  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
413  LHS = GetScalarizedVector(LHS);
414  RHS = GetScalarizedVector(RHS);
415  } else {
416  EVT VT = OpVT.getVectorElementType();
417  LHS = DAG.getNode(
418  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
419  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
420  RHS = DAG.getNode(
421  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
422  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
423  }
424 
425  // Turn it into a scalar SETCC.
426  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
427  N->getOperand(2));
428  // Vectors may have a different boolean contents to scalars. Promote the
429  // value appropriately.
430  ISD::NodeType ExtendCode =
432  return DAG.getNode(ExtendCode, DL, NVT, Res);
433 }
434 
435 
436 //===----------------------------------------------------------------------===//
437 // Operand Vector Scalarization <1 x ty> -> ty.
438 //===----------------------------------------------------------------------===//
439 
440 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
441  DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
442  N->dump(&DAG);
443  dbgs() << "\n");
444  SDValue Res = SDValue();
445 
446  if (!Res.getNode()) {
447  switch (N->getOpcode()) {
448  default:
449 #ifndef NDEBUG
450  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
451  N->dump(&DAG);
452  dbgs() << "\n";
453 #endif
454  report_fatal_error("Do not know how to scalarize this operator's "
455  "operand!\n");
456  case ISD::BITCAST:
457  Res = ScalarizeVecOp_BITCAST(N);
458  break;
459  case ISD::ANY_EXTEND:
460  case ISD::ZERO_EXTEND:
461  case ISD::SIGN_EXTEND:
462  case ISD::TRUNCATE:
463  case ISD::FP_TO_SINT:
464  case ISD::FP_TO_UINT:
465  case ISD::SINT_TO_FP:
466  case ISD::UINT_TO_FP:
467  Res = ScalarizeVecOp_UnaryOp(N);
468  break;
469  case ISD::CONCAT_VECTORS:
470  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
471  break;
473  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
474  break;
475  case ISD::VSELECT:
476  Res = ScalarizeVecOp_VSELECT(N);
477  break;
478  case ISD::SETCC:
479  Res = ScalarizeVecOp_VSETCC(N);
480  break;
481  case ISD::STORE:
482  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
483  break;
484  case ISD::FP_ROUND:
485  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
486  break;
487  }
488  }
489 
490  // If the result is null, the sub-method took care of registering results etc.
491  if (!Res.getNode()) return false;
492 
493  // If the result is N, the sub-method updated N in place. Tell the legalizer
494  // core about this.
495  if (Res.getNode() == N)
496  return true;
497 
498  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
499  "Invalid operand expansion");
500 
501  ReplaceValueWith(SDValue(N, 0), Res);
502  return false;
503 }
504 
505 /// If the value to convert is a vector that needs to be scalarized, it must be
506 /// <1 x ty>. Convert the element instead.
507 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
508  SDValue Elt = GetScalarizedVector(N->getOperand(0));
509  return DAG.getNode(ISD::BITCAST, SDLoc(N),
510  N->getValueType(0), Elt);
511 }
512 
513 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
514 /// Do the operation on the element instead.
515 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
517  "Unexpected vector type!");
518  SDValue Elt = GetScalarizedVector(N->getOperand(0));
519  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
520  N->getValueType(0).getScalarType(), Elt);
521  // Revectorize the result so the types line up with what the uses of this
522  // expression expect.
523  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Op);
524 }
525 
526 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
527 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
529  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
530  Ops[i] = GetScalarizedVector(N->getOperand(i));
531  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
532 }
533 
534 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
535 /// so just return the element, ignoring the index.
536 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
537  EVT VT = N->getValueType(0);
538  SDValue Res = GetScalarizedVector(N->getOperand(0));
539  if (Res.getValueType() != VT)
540  Res = VT.isFloatingPoint()
541  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
542  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
543  return Res;
544 }
545 
546 /// If the input condition is a vector that needs to be scalarized, it must be
547 /// <1 x i1>, so just convert to a normal ISD::SELECT
548 /// (still with vector output type since that was acceptable if we got here).
549 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
550  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
551  EVT VT = N->getValueType(0);
552 
553  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
554  N->getOperand(2));
555 }
556 
557 /// If the operand is a vector that needs to be scalarized then the
558 /// result must be v1i1, so just convert to a scalar SETCC and wrap
559 /// with a scalar_to_vector since the res type is legal if we got here
560 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
561  assert(N->getValueType(0).isVector() &&
562  N->getOperand(0).getValueType().isVector() &&
563  "Operand types must be vectors");
564  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
565 
566  EVT VT = N->getValueType(0);
567  SDValue LHS = GetScalarizedVector(N->getOperand(0));
568  SDValue RHS = GetScalarizedVector(N->getOperand(1));
569 
570  EVT OpVT = N->getOperand(0).getValueType();
571  EVT NVT = VT.getVectorElementType();
572  SDLoc DL(N);
573  // Turn it into a scalar SETCC.
574  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
575  N->getOperand(2));
576 
577  // Vectors may have a different boolean contents to scalars. Promote the
578  // value appropriately.
579  ISD::NodeType ExtendCode =
581 
582  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
583 
584  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
585 }
586 
587 /// If the value to store is a vector that needs to be scalarized, it must be
588 /// <1 x ty>. Just store the element.
589 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
590  assert(N->isUnindexed() && "Indexed store of one-element vector?");
591  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
592  SDLoc dl(N);
593 
594  if (N->isTruncatingStore())
595  return DAG.getTruncStore(
596  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
597  N->getBasePtr(), N->getPointerInfo(),
599  N->getMemOperand()->getFlags(), N->getAAInfo());
600 
601  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
602  N->getBasePtr(), N->getPointerInfo(),
604  N->getAAInfo());
605 }
606 
607 /// If the value to round is a vector that needs to be scalarized, it must be
608 /// <1 x ty>. Convert the element instead.
609 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
610  SDValue Elt = GetScalarizedVector(N->getOperand(0));
611  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
612  N->getValueType(0).getVectorElementType(), Elt,
613  N->getOperand(1));
614  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
615 }
616 
617 //===----------------------------------------------------------------------===//
618 // Result Vector Splitting
619 //===----------------------------------------------------------------------===//
620 
621 /// This method is called when the specified result of the specified node is
622 /// found to need vector splitting. At this point, the node may also have
623 /// invalid operands or may have other results that need legalization, we just
624 /// know that (at least) one result needs vector splitting.
625 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
626  DEBUG(dbgs() << "Split node result: ";
627  N->dump(&DAG);
628  dbgs() << "\n");
629  SDValue Lo, Hi;
630 
631  // See if the target wants to custom expand this node.
632  if (CustomLowerNode(N, N->getValueType(ResNo), true))
633  return;
634 
635  switch (N->getOpcode()) {
636  default:
637 #ifndef NDEBUG
638  dbgs() << "SplitVectorResult #" << ResNo << ": ";
639  N->dump(&DAG);
640  dbgs() << "\n";
641 #endif
642  report_fatal_error("Do not know how to split the result of this "
643  "operator!\n");
644 
645  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
646  case ISD::VSELECT:
647  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
648  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
649  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
650  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
651  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
652  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
653  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
654  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
655  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
656  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
657  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
658  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
659  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
660  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
661  case ISD::LOAD:
662  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
663  break;
664  case ISD::MLOAD:
665  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
666  break;
667  case ISD::MGATHER:
668  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
669  break;
670  case ISD::SETCC:
671  SplitVecRes_SETCC(N, Lo, Hi);
672  break;
673  case ISD::VECTOR_SHUFFLE:
674  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
675  break;
676 
680  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
681  break;
682 
683  case ISD::BITREVERSE:
684  case ISD::BSWAP:
685  case ISD::CTLZ:
686  case ISD::CTTZ:
689  case ISD::CTPOP:
690  case ISD::FABS:
691  case ISD::FCEIL:
692  case ISD::FCOS:
693  case ISD::FEXP:
694  case ISD::FEXP2:
695  case ISD::FFLOOR:
696  case ISD::FLOG:
697  case ISD::FLOG10:
698  case ISD::FLOG2:
699  case ISD::FNEARBYINT:
700  case ISD::FNEG:
701  case ISD::FP_EXTEND:
702  case ISD::FP_ROUND:
703  case ISD::FP_TO_SINT:
704  case ISD::FP_TO_UINT:
705  case ISD::FRINT:
706  case ISD::FROUND:
707  case ISD::FSIN:
708  case ISD::FSQRT:
709  case ISD::FTRUNC:
710  case ISD::SINT_TO_FP:
711  case ISD::TRUNCATE:
712  case ISD::UINT_TO_FP:
713  case ISD::FCANONICALIZE:
714  SplitVecRes_UnaryOp(N, Lo, Hi);
715  break;
716 
717  case ISD::ANY_EXTEND:
718  case ISD::SIGN_EXTEND:
719  case ISD::ZERO_EXTEND:
720  SplitVecRes_ExtendOp(N, Lo, Hi);
721  break;
722 
723  case ISD::ADD:
724  case ISD::SUB:
725  case ISD::MUL:
726  case ISD::MULHS:
727  case ISD::MULHU:
728  case ISD::FADD:
729  case ISD::FSUB:
730  case ISD::FMUL:
731  case ISD::FMINNUM:
732  case ISD::FMAXNUM:
733  case ISD::FMINNAN:
734  case ISD::FMAXNAN:
735  case ISD::SDIV:
736  case ISD::UDIV:
737  case ISD::FDIV:
738  case ISD::FPOW:
739  case ISD::AND:
740  case ISD::OR:
741  case ISD::XOR:
742  case ISD::SHL:
743  case ISD::SRA:
744  case ISD::SRL:
745  case ISD::UREM:
746  case ISD::SREM:
747  case ISD::FREM:
748  case ISD::SMIN:
749  case ISD::SMAX:
750  case ISD::UMIN:
751  case ISD::UMAX:
752  SplitVecRes_BinOp(N, Lo, Hi);
753  break;
754  case ISD::FMA:
755  SplitVecRes_TernaryOp(N, Lo, Hi);
756  break;
757  }
758 
759  // If Lo/Hi is null, the sub-method took care of registering results etc.
760  if (Lo.getNode())
761  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
762 }
763 
764 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
765  SDValue &Hi) {
766  SDValue LHSLo, LHSHi;
767  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
768  SDValue RHSLo, RHSHi;
769  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
770  SDLoc dl(N);
771 
772  const SDNodeFlags Flags = N->getFlags();
773  unsigned Opcode = N->getOpcode();
774  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
775  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
776 }
777 
778 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
779  SDValue &Hi) {
780  SDValue Op0Lo, Op0Hi;
781  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
782  SDValue Op1Lo, Op1Hi;
783  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
784  SDValue Op2Lo, Op2Hi;
785  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
786  SDLoc dl(N);
787 
788  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
789  Op0Lo, Op1Lo, Op2Lo);
790  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
791  Op0Hi, Op1Hi, Op2Hi);
792 }
793 
794 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
795  SDValue &Hi) {
796  // We know the result is a vector. The input may be either a vector or a
797  // scalar value.
798  EVT LoVT, HiVT;
799  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
800  SDLoc dl(N);
801 
802  SDValue InOp = N->getOperand(0);
803  EVT InVT = InOp.getValueType();
804 
805  // Handle some special cases efficiently.
806  switch (getTypeAction(InVT)) {
813  break;
816  // A scalar to vector conversion, where the scalar needs expansion.
817  // If the vector is being split in two then we can just convert the
818  // expanded pieces.
819  if (LoVT == HiVT) {
820  GetExpandedOp(InOp, Lo, Hi);
821  if (DAG.getDataLayout().isBigEndian())
822  std::swap(Lo, Hi);
823  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
824  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
825  return;
826  }
827  break;
829  // If the input is a vector that needs to be split, convert each split
830  // piece of the input now.
831  GetSplitVector(InOp, Lo, Hi);
832  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
833  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
834  return;
835  }
836 
837  // In the general case, convert the input to an integer and split it by hand.
838  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
839  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
840  if (DAG.getDataLayout().isBigEndian())
841  std::swap(LoIntVT, HiIntVT);
842 
843  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
844 
845  if (DAG.getDataLayout().isBigEndian())
846  std::swap(Lo, Hi);
847  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
848  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
849 }
850 
851 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
852  SDValue &Hi) {
853  EVT LoVT, HiVT;
854  SDLoc dl(N);
855  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
856  unsigned LoNumElts = LoVT.getVectorNumElements();
857  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
858  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
859 
860  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
861  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
862 }
863 
864 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
865  SDValue &Hi) {
866  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
867  SDLoc dl(N);
868  unsigned NumSubvectors = N->getNumOperands() / 2;
869  if (NumSubvectors == 1) {
870  Lo = N->getOperand(0);
871  Hi = N->getOperand(1);
872  return;
873  }
874 
875  EVT LoVT, HiVT;
876  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
877 
878  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
879  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
880 
881  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
882  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
883 }
884 
885 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
886  SDValue &Hi) {
887  SDValue Vec = N->getOperand(0);
888  SDValue Idx = N->getOperand(1);
889  SDLoc dl(N);
890 
891  EVT LoVT, HiVT;
892  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
893 
894  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
895  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
896  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
897  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
898  TLI.getVectorIdxTy(DAG.getDataLayout())));
899 }
900 
901 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
902  SDValue &Hi) {
903  SDValue Vec = N->getOperand(0);
904  SDValue SubVec = N->getOperand(1);
905  SDValue Idx = N->getOperand(2);
906  SDLoc dl(N);
907  GetSplitVector(Vec, Lo, Hi);
908 
909  EVT VecVT = Vec.getValueType();
910  unsigned VecElems = VecVT.getVectorNumElements();
911  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
912 
913  // If we know the index is 0, and we know the subvector doesn't cross the
914  // boundary between the halves, we can avoid spilling the vector, and insert
915  // into the lower half of the split vector directly.
916  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
917  // the index is constant and there is no boundary crossing. But those cases
918  // don't seem to get hit in practice.
919  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
920  unsigned IdxVal = ConstIdx->getZExtValue();
921  if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
922  EVT LoVT, HiVT;
923  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
924  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
925  return;
926  }
927  }
928 
929  // Spill the vector to the stack.
930  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
931  SDValue Store =
932  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
933 
934  // Store the new subvector into the specified index.
935  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
936  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
937  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
938  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
939 
940  // Load the Lo part from the stack slot.
941  Lo =
942  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
943 
944  // Increment the pointer to the other part.
945  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
946  StackPtr =
947  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
948  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
949 
950  // Load the Hi part from the stack slot.
951  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
952  MinAlign(Alignment, IncrementSize));
953 }
954 
955 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
956  SDValue &Hi) {
957  SDLoc dl(N);
958  GetSplitVector(N->getOperand(0), Lo, Hi);
959  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
960  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
961 }
962 
963 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
964  SDValue &Hi) {
965  SDValue LHSLo, LHSHi;
966  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
967  SDLoc DL(N);
968 
969  SDValue RHSLo, RHSHi;
970  SDValue RHS = N->getOperand(1);
971  EVT RHSVT = RHS.getValueType();
972  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
973  GetSplitVector(RHS, RHSLo, RHSHi);
974  else
975  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
976 
977 
978  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
979  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
980 }
981 
982 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
983  SDValue &Hi) {
984  SDValue LHSLo, LHSHi;
985  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
986  SDLoc dl(N);
987 
988  EVT LoVT, HiVT;
989  std::tie(LoVT, HiVT) =
990  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
991 
992  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
993  DAG.getValueType(LoVT));
994  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
995  DAG.getValueType(HiVT));
996 }
997 
998 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
999  SDValue &Hi) {
1000  unsigned Opcode = N->getOpcode();
1001  SDValue N0 = N->getOperand(0);
1002 
1003  SDLoc dl(N);
1004  SDValue InLo, InHi;
1005 
1006  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1007  GetSplitVector(N0, InLo, InHi);
1008  else
1009  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1010 
1011  EVT InLoVT = InLo.getValueType();
1012  unsigned InNumElements = InLoVT.getVectorNumElements();
1013 
1014  EVT OutLoVT, OutHiVT;
1015  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1016  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1017  assert((2 * OutNumElements) <= InNumElements &&
1018  "Illegal extend vector in reg split");
1019 
1020  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1021  // input vector (i.e. we only use InLo):
1022  // OutLo will extend the first OutNumElements from InLo.
1023  // OutHi will extend the next OutNumElements from InLo.
1024 
1025  // Shuffle the elements from InLo for OutHi into the bottom elements to
1026  // create a 'fake' InHi.
1027  SmallVector<int, 8> SplitHi(InNumElements, -1);
1028  for (unsigned i = 0; i != OutNumElements; ++i)
1029  SplitHi[i] = i + OutNumElements;
1030  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1031 
1032  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1033  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1034 }
1035 
1036 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1037  SDValue &Hi) {
1038  SDValue Vec = N->getOperand(0);
1039  SDValue Elt = N->getOperand(1);
1040  SDValue Idx = N->getOperand(2);
1041  SDLoc dl(N);
1042  GetSplitVector(Vec, Lo, Hi);
1043 
1044  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1045  unsigned IdxVal = CIdx->getZExtValue();
1046  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1047  if (IdxVal < LoNumElts)
1048  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1049  Lo.getValueType(), Lo, Elt, Idx);
1050  else
1051  Hi =
1052  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1053  DAG.getConstant(IdxVal - LoNumElts, dl,
1054  TLI.getVectorIdxTy(DAG.getDataLayout())));
1055  return;
1056  }
1057 
1058  // See if the target wants to custom expand this node.
1059  if (CustomLowerNode(N, N->getValueType(0), true))
1060  return;
1061 
1062  // Make the vector elements byte-addressable if they aren't already.
1063  EVT VecVT = Vec.getValueType();
1064  EVT EltVT = VecVT.getVectorElementType();
1065  if (VecVT.getScalarSizeInBits() < 8) {
1066  EltVT = MVT::i8;
1067  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1068  VecVT.getVectorNumElements());
1069  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1070  // Extend the element type to match if needed.
1071  if (EltVT.bitsGT(Elt.getValueType()))
1072  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1073  }
1074 
1075  // Spill the vector to the stack.
1076  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1077  auto &MF = DAG.getMachineFunction();
1078  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1079  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1080  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1081 
1082  // Store the new element. This may be larger than the vector element type,
1083  // so use a truncating store.
1084  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1085  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1086  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1087  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1089 
1090  EVT LoVT, HiVT;
1091  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1092 
1093  // Load the Lo part from the stack slot.
1094  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1095 
1096  // Increment the pointer to the other part.
1097  unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1098  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1099  DAG.getConstant(IncrementSize, dl,
1100  StackPtr.getValueType()));
1101 
1102  // Load the Hi part from the stack slot.
1103  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1104  PtrInfo.getWithOffset(IncrementSize),
1105  MinAlign(Alignment, IncrementSize));
1106 
1107  // If we adjusted the original type, we need to truncate the results.
1108  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1109  if (LoVT != Lo.getValueType())
1110  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1111  if (HiVT != Hi.getValueType())
1112  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1113 }
1114 
1115 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1116  SDValue &Hi) {
1117  EVT LoVT, HiVT;
1118  SDLoc dl(N);
1119  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1120  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1121  Hi = DAG.getUNDEF(HiVT);
1122 }
1123 
1124 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1125  SDValue &Hi) {
1126  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1127  EVT LoVT, HiVT;
1128  SDLoc dl(LD);
1129  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1130 
1131  ISD::LoadExtType ExtType = LD->getExtensionType();
1132  SDValue Ch = LD->getChain();
1133  SDValue Ptr = LD->getBasePtr();
1134  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1135  EVT MemoryVT = LD->getMemoryVT();
1136  unsigned Alignment = LD->getOriginalAlignment();
1137  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1138  AAMDNodes AAInfo = LD->getAAInfo();
1139 
1140  EVT LoMemVT, HiMemVT;
1141  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1142 
1143  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1144  LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1145 
1146  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1147  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1148  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1149  LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1150  Alignment, MMOFlags, AAInfo);
1151 
1152  // Build a factor node to remember that this load is independent of the
1153  // other one.
1154  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1155  Hi.getValue(1));
1156 
1157  // Legalize the chain result - switch anything that used the old chain to
1158  // use the new one.
1159  ReplaceValueWith(SDValue(LD, 1), Ch);
1160 }
1161 
1162 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1163  SDValue &Lo, SDValue &Hi) {
1164  EVT LoVT, HiVT;
1165  SDLoc dl(MLD);
1166  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1167 
1168  SDValue Ch = MLD->getChain();
1169  SDValue Ptr = MLD->getBasePtr();
1170  SDValue Mask = MLD->getMask();
1171  SDValue Src0 = MLD->getSrc0();
1172  unsigned Alignment = MLD->getOriginalAlignment();
1173  ISD::LoadExtType ExtType = MLD->getExtensionType();
1174 
1175  // if Alignment is equal to the vector size,
1176  // take the half of it for the second part
1177  unsigned SecondHalfAlignment =
1178  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1179  Alignment/2 : Alignment;
1180 
1181  // Split Mask operand
1182  SDValue MaskLo, MaskHi;
1183  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1184  GetSplitVector(Mask, MaskLo, MaskHi);
1185  else
1186  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1187 
1188  EVT MemoryVT = MLD->getMemoryVT();
1189  EVT LoMemVT, HiMemVT;
1190  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1191 
1192  SDValue Src0Lo, Src0Hi;
1193  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1194  GetSplitVector(Src0, Src0Lo, Src0Hi);
1195  else
1196  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1197 
1199  getMachineMemOperand(MLD->getPointerInfo(),
1201  Alignment, MLD->getAAInfo(), MLD->getRanges());
1202 
1203  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
1204  ExtType, MLD->isExpandingLoad());
1205 
1206  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1207  MLD->isExpandingLoad());
1208 
1209  MMO = DAG.getMachineFunction().
1210  getMachineMemOperand(MLD->getPointerInfo(),
1212  SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());
1213 
1214  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
1215  ExtType, MLD->isExpandingLoad());
1216 
1217 
1218  // Build a factor node to remember that this load is independent of the
1219  // other one.
1220  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1221  Hi.getValue(1));
1222 
1223  // Legalize the chain result - switch anything that used the old chain to
1224  // use the new one.
1225  ReplaceValueWith(SDValue(MLD, 1), Ch);
1226 
1227 }
1228 
1229 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1230  SDValue &Lo, SDValue &Hi) {
1231  EVT LoVT, HiVT;
1232  SDLoc dl(MGT);
1233  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1234 
1235  SDValue Ch = MGT->getChain();
1236  SDValue Ptr = MGT->getBasePtr();
1237  SDValue Mask = MGT->getMask();
1238  SDValue Src0 = MGT->getValue();
1239  SDValue Index = MGT->getIndex();
1240  SDValue Scale = MGT->getScale();
1241  unsigned Alignment = MGT->getOriginalAlignment();
1242 
1243  // Split Mask operand
1244  SDValue MaskLo, MaskHi;
1245  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1246  GetSplitVector(Mask, MaskLo, MaskHi);
1247  else
1248  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1249 
1250  EVT MemoryVT = MGT->getMemoryVT();
1251  EVT LoMemVT, HiMemVT;
1252  // Split MemoryVT
1253  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1254 
1255  SDValue Src0Lo, Src0Hi;
1256  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1257  GetSplitVector(Src0, Src0Lo, Src0Hi);
1258  else
1259  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1260 
1261  SDValue IndexHi, IndexLo;
1262  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1263  GetSplitVector(Index, IndexLo, IndexHi);
1264  else
1265  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1266 
1268  getMachineMemOperand(MGT->getPointerInfo(),
1270  Alignment, MGT->getAAInfo(), MGT->getRanges());
1271 
1272  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
1273  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1274  MMO);
1275 
1276  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
1277  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1278  MMO);
1279 
1280  // Build a factor node to remember that this load is independent of the
1281  // other one.
1282  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1283  Hi.getValue(1));
1284 
1285  // Legalize the chain result - switch anything that used the old chain to
1286  // use the new one.
1287  ReplaceValueWith(SDValue(MGT, 1), Ch);
1288 }
1289 
1290 
1291 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1292  assert(N->getValueType(0).isVector() &&
1293  N->getOperand(0).getValueType().isVector() &&
1294  "Operand types must be vectors");
1295 
1296  EVT LoVT, HiVT;
1297  SDLoc DL(N);
1298  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1299 
1300  // If the input also splits, handle it directly. Otherwise split it by hand.
1301  SDValue LL, LH, RL, RH;
1302  if (getTypeAction(N->getOperand(0).getValueType()) ==
1304  GetSplitVector(N->getOperand(0), LL, LH);
1305  else
1306  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1307 
1308  if (getTypeAction(N->getOperand(1).getValueType()) ==
1310  GetSplitVector(N->getOperand(1), RL, RH);
1311  else
1312  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1313 
1314  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1315  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1316 }
1317 
1318 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1319  SDValue &Hi) {
1320  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1321  EVT LoVT, HiVT;
1322  SDLoc dl(N);
1323  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1324 
1325  // If the input also splits, handle it directly for a compile time speedup.
1326  // Otherwise split it by hand.
1327  EVT InVT = N->getOperand(0).getValueType();
1328  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1329  GetSplitVector(N->getOperand(0), Lo, Hi);
1330  else
1331  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1332 
1333  if (N->getOpcode() == ISD::FP_ROUND) {
1334  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1335  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1336  } else {
1337  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1338  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1339  }
1340 }
1341 
1342 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1343  SDValue &Hi) {
1344  SDLoc dl(N);
1345  EVT SrcVT = N->getOperand(0).getValueType();
1346  EVT DestVT = N->getValueType(0);
1347  EVT LoVT, HiVT;
1348  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1349 
1350  // We can do better than a generic split operation if the extend is doing
1351  // more than just doubling the width of the elements and the following are
1352  // true:
1353  // - The number of vector elements is even,
1354  // - the source type is legal,
1355  // - the type of a split source is illegal,
1356  // - the type of an extended (by doubling element size) source is legal, and
1357  // - the type of that extended source when split is legal.
1358  //
1359  // This won't necessarily completely legalize the operation, but it will
1360  // more effectively move in the right direction and prevent falling down
1361  // to scalarization in many cases due to the input vector being split too
1362  // far.
1363  unsigned NumElements = SrcVT.getVectorNumElements();
1364  if ((NumElements & 1) == 0 &&
1365  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1366  LLVMContext &Ctx = *DAG.getContext();
1367  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1368  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1369 
1370  EVT SplitLoVT, SplitHiVT;
1371  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1372  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1373  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1374  DEBUG(dbgs() << "Split vector extend via incremental extend:";
1375  N->dump(&DAG); dbgs() << "\n");
1376  // Extend the source vector by one step.
1377  SDValue NewSrc =
1378  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1379  // Get the low and high halves of the new, extended one step, vector.
1380  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1381  // Extend those vector halves the rest of the way.
1382  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1383  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1384  return;
1385  }
1386  }
1387  // Fall back to the generic unary operator splitting otherwise.
1388  SplitVecRes_UnaryOp(N, Lo, Hi);
1389 }
1390 
1391 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1392  SDValue &Lo, SDValue &Hi) {
1393  // The low and high parts of the original input give four input vectors.
1394  SDValue Inputs[4];
1395  SDLoc dl(N);
1396  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1397  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1398  EVT NewVT = Inputs[0].getValueType();
1399  unsigned NewElts = NewVT.getVectorNumElements();
1400 
1401  // If Lo or Hi uses elements from at most two of the four input vectors, then
1402  // express it as a vector shuffle of those two inputs. Otherwise extract the
1403  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1405  for (unsigned High = 0; High < 2; ++High) {
1406  SDValue &Output = High ? Hi : Lo;
1407 
1408  // Build a shuffle mask for the output, discovering on the fly which
1409  // input vectors to use as shuffle operands (recorded in InputUsed).
1410  // If building a suitable shuffle vector proves too hard, then bail
1411  // out with useBuildVector set.
1412  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1413  unsigned FirstMaskIdx = High * NewElts;
1414  bool useBuildVector = false;
1415  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1416  // The mask element. This indexes into the input.
1417  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1418 
1419  // The input vector this mask element indexes into.
1420  unsigned Input = (unsigned)Idx / NewElts;
1421 
1422  if (Input >= array_lengthof(Inputs)) {
1423  // The mask element does not index into any input vector.
1424  Ops.push_back(-1);
1425  continue;
1426  }
1427 
1428  // Turn the index into an offset from the start of the input vector.
1429  Idx -= Input * NewElts;
1430 
1431  // Find or create a shuffle vector operand to hold this input.
1432  unsigned OpNo;
1433  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1434  if (InputUsed[OpNo] == Input) {
1435  // This input vector is already an operand.
1436  break;
1437  } else if (InputUsed[OpNo] == -1U) {
1438  // Create a new operand for this input vector.
1439  InputUsed[OpNo] = Input;
1440  break;
1441  }
1442  }
1443 
1444  if (OpNo >= array_lengthof(InputUsed)) {
1445  // More than two input vectors used! Give up on trying to create a
1446  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1447  useBuildVector = true;
1448  break;
1449  }
1450 
1451  // Add the mask index for the new shuffle vector.
1452  Ops.push_back(Idx + OpNo * NewElts);
1453  }
1454 
1455  if (useBuildVector) {
1456  EVT EltVT = NewVT.getVectorElementType();
1458 
1459  // Extract the input elements by hand.
1460  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1461  // The mask element. This indexes into the input.
1462  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1463 
1464  // The input vector this mask element indexes into.
1465  unsigned Input = (unsigned)Idx / NewElts;
1466 
1467  if (Input >= array_lengthof(Inputs)) {
1468  // The mask element is "undef" or indexes off the end of the input.
1469  SVOps.push_back(DAG.getUNDEF(EltVT));
1470  continue;
1471  }
1472 
1473  // Turn the index into an offset from the start of the input vector.
1474  Idx -= Input * NewElts;
1475 
1476  // Extract the vector element by hand.
1477  SVOps.push_back(DAG.getNode(
1478  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1479  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1480  }
1481 
1482  // Construct the Lo/Hi output using a BUILD_VECTOR.
1483  Output = DAG.getBuildVector(NewVT, dl, SVOps);
1484  } else if (InputUsed[0] == -1U) {
1485  // No input vectors were used! The result is undefined.
1486  Output = DAG.getUNDEF(NewVT);
1487  } else {
1488  SDValue Op0 = Inputs[InputUsed[0]];
1489  // If only one input was used, use an undefined vector for the other.
1490  SDValue Op1 = InputUsed[1] == -1U ?
1491  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1492  // At least one input vector was used. Create a new shuffle vector.
1493  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1494  }
1495 
1496  Ops.clear();
1497  }
1498 }
1499 
1500 
1501 //===----------------------------------------------------------------------===//
1502 // Operand Vector Splitting
1503 //===----------------------------------------------------------------------===//
1504 
1505 /// This method is called when the specified operand of the specified node is
1506 /// found to need vector splitting. At this point, all of the result types of
1507 /// the node are known to be legal, but other operands of the node may need
1508 /// legalization as well as the specified one.
1509 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1510  DEBUG(dbgs() << "Split node operand: ";
1511  N->dump(&DAG);
1512  dbgs() << "\n");
1513  SDValue Res = SDValue();
1514 
1515  // See if the target wants to custom split this node.
1516  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1517  return false;
1518 
1519  if (!Res.getNode()) {
1520  switch (N->getOpcode()) {
1521  default:
1522 #ifndef NDEBUG
1523  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1524  N->dump(&DAG);
1525  dbgs() << "\n";
1526 #endif
1527  report_fatal_error("Do not know how to split this operator's "
1528  "operand!\n");
1529 
1530  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1531  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1532  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1533  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1534  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1535  case ISD::TRUNCATE:
1536  Res = SplitVecOp_TruncateHelper(N);
1537  break;
1538  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1539  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1540  case ISD::STORE:
1541  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1542  break;
1543  case ISD::MSTORE:
1544  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1545  break;
1546  case ISD::MSCATTER:
1547  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1548  break;
1549  case ISD::MGATHER:
1550  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1551  break;
1552  case ISD::VSELECT:
1553  Res = SplitVecOp_VSELECT(N, OpNo);
1554  break;
1555  case ISD::FP_TO_SINT:
1556  case ISD::FP_TO_UINT:
1557  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1558  Res = SplitVecOp_TruncateHelper(N);
1559  else
1560  Res = SplitVecOp_UnaryOp(N);
1561  break;
1562  case ISD::SINT_TO_FP:
1563  case ISD::UINT_TO_FP:
1564  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1565  Res = SplitVecOp_TruncateHelper(N);
1566  else
1567  Res = SplitVecOp_UnaryOp(N);
1568  break;
1569  case ISD::CTTZ:
1570  case ISD::CTLZ:
1571  case ISD::CTPOP:
1572  case ISD::FP_EXTEND:
1573  case ISD::SIGN_EXTEND:
1574  case ISD::ZERO_EXTEND:
1575  case ISD::ANY_EXTEND:
1576  case ISD::FTRUNC:
1577  case ISD::FCANONICALIZE:
1578  Res = SplitVecOp_UnaryOp(N);
1579  break;
1580 
1584  Res = SplitVecOp_ExtVecInRegOp(N);
1585  break;
1586 
1587  case ISD::VECREDUCE_FADD:
1588  case ISD::VECREDUCE_FMUL:
1589  case ISD::VECREDUCE_ADD:
1590  case ISD::VECREDUCE_MUL:
1591  case ISD::VECREDUCE_AND:
1592  case ISD::VECREDUCE_OR:
1593  case ISD::VECREDUCE_XOR:
1594  case ISD::VECREDUCE_SMAX:
1595  case ISD::VECREDUCE_SMIN:
1596  case ISD::VECREDUCE_UMAX:
1597  case ISD::VECREDUCE_UMIN:
1598  case ISD::VECREDUCE_FMAX:
1599  case ISD::VECREDUCE_FMIN:
1600  Res = SplitVecOp_VECREDUCE(N, OpNo);
1601  break;
1602  }
1603  }
1604 
1605  // If the result is null, the sub-method took care of registering results etc.
1606  if (!Res.getNode()) return false;
1607 
1608  // If the result is N, the sub-method updated N in place. Tell the legalizer
1609  // core about this.
1610  if (Res.getNode() == N)
1611  return true;
1612 
1613  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1614  "Invalid operand expansion");
1615 
1616  ReplaceValueWith(SDValue(N, 0), Res);
1617  return false;
1618 }
1619 
1620 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1621  // The only possibility for an illegal operand is the mask, since result type
1622  // legalization would have handled this node already otherwise.
1623  assert(OpNo == 0 && "Illegal operand must be mask");
1624 
1625  SDValue Mask = N->getOperand(0);
1626  SDValue Src0 = N->getOperand(1);
1627  SDValue Src1 = N->getOperand(2);
1628  EVT Src0VT = Src0.getValueType();
1629  SDLoc DL(N);
1630  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1631 
1632  SDValue Lo, Hi;
1633  GetSplitVector(N->getOperand(0), Lo, Hi);
1634  assert(Lo.getValueType() == Hi.getValueType() &&
1635  "Lo and Hi have differing types");
1636 
1637  EVT LoOpVT, HiOpVT;
1638  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1639  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1640 
1641  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1642  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1643  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1644  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1645 
1646  SDValue LoSelect =
1647  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1648  SDValue HiSelect =
1649  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1650 
1651  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1652 }
1653 
1654 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
1655  EVT ResVT = N->getValueType(0);
1656  SDValue Lo, Hi;
1657  SDLoc dl(N);
1658 
1659  SDValue VecOp = N->getOperand(OpNo);
1660  EVT VecVT = VecOp.getValueType();
1661  assert(VecVT.isVector() && "Can only split reduce vector operand");
1662  GetSplitVector(VecOp, Lo, Hi);
1663  EVT LoOpVT, HiOpVT;
1664  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
1665 
1666  bool NoNaN = N->getFlags().hasNoNaNs();
1667  unsigned CombineOpc = 0;
1668  switch (N->getOpcode()) {
1669  case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
1670  case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
1671  case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
1672  case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
1673  case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
1674  case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
1675  case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
1676  case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
1677  case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
1678  case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
1679  case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
1680  case ISD::VECREDUCE_FMAX:
1681  CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
1682  break;
1683  case ISD::VECREDUCE_FMIN:
1684  CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
1685  break;
1686  default:
1687  llvm_unreachable("Unexpected reduce ISD node");
1688  }
1689 
1690  // Use the appropriate scalar instruction on the split subvectors before
1691  // reducing the now partially reduced smaller vector.
1692  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi);
1693  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial);
1694 }
1695 
1696 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1697  // The result has a legal vector type, but the input needs splitting.
1698  EVT ResVT = N->getValueType(0);
1699  SDValue Lo, Hi;
1700  SDLoc dl(N);
1701  GetSplitVector(N->getOperand(0), Lo, Hi);
1702  EVT InVT = Lo.getValueType();
1703 
1704  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1705  InVT.getVectorNumElements());
1706 
1707  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1708  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1709 
1710  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1711 }
1712 
1713 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1714  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1715  // end up being split all the way down to individual components. Convert the
1716  // split pieces into integers and reassemble.
1717  SDValue Lo, Hi;
1718  GetSplitVector(N->getOperand(0), Lo, Hi);
1719  Lo = BitConvertToInteger(Lo);
1720  Hi = BitConvertToInteger(Hi);
1721 
1722  if (DAG.getDataLayout().isBigEndian())
1723  std::swap(Lo, Hi);
1724 
1725  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1726  JoinIntegers(Lo, Hi));
1727 }
1728 
1729 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1730  // We know that the extracted result type is legal.
1731  EVT SubVT = N->getValueType(0);
1732  SDValue Idx = N->getOperand(1);
1733  SDLoc dl(N);
1734  SDValue Lo, Hi;
1735  GetSplitVector(N->getOperand(0), Lo, Hi);
1736 
1737  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1738  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1739 
1740  if (IdxVal < LoElts) {
1741  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1742  "Extracted subvector crosses vector split!");
1743  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1744  } else {
1745  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1746  DAG.getConstant(IdxVal - LoElts, dl,
1747  Idx.getValueType()));
1748  }
1749 }
1750 
1751 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1752  SDValue Vec = N->getOperand(0);
1753  SDValue Idx = N->getOperand(1);
1754  EVT VecVT = Vec.getValueType();
1755 
1756  if (isa<ConstantSDNode>(Idx)) {
1757  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1758  assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1759 
1760  SDValue Lo, Hi;
1761  GetSplitVector(Vec, Lo, Hi);
1762 
1763  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1764 
1765  if (IdxVal < LoElts)
1766  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1767  return SDValue(DAG.UpdateNodeOperands(N, Hi,
1768  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
1769  Idx.getValueType())), 0);
1770  }
1771 
1772  // See if the target wants to custom expand this node.
1773  if (CustomLowerNode(N, N->getValueType(0), true))
1774  return SDValue();
1775 
1776  // Make the vector elements byte-addressable if they aren't already.
1777  SDLoc dl(N);
1778  EVT EltVT = VecVT.getVectorElementType();
1779  if (VecVT.getScalarSizeInBits() < 8) {
1780  EltVT = MVT::i8;
1781  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1782  VecVT.getVectorNumElements());
1783  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1784  }
1785 
1786  // Store the vector to the stack.
1787  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1788  auto &MF = DAG.getMachineFunction();
1789  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1790  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1791  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1792 
1793  // Load back the required element.
1794  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1795  return DAG.getExtLoad(
1796  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1798 }
1799 
1800 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
1801  SDValue Lo, Hi;
1802 
1803  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
1804  // splitting the result has the same effect as splitting the input operand.
1805  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1806 
1807  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
1808 }
1809 
1810 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
1811  unsigned OpNo) {
1812  EVT LoVT, HiVT;
1813  SDLoc dl(MGT);
1814  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1815 
1816  SDValue Ch = MGT->getChain();
1817  SDValue Ptr = MGT->getBasePtr();
1818  SDValue Index = MGT->getIndex();
1819  SDValue Scale = MGT->getScale();
1820  SDValue Mask = MGT->getMask();
1821  SDValue Src0 = MGT->getValue();
1822  unsigned Alignment = MGT->getOriginalAlignment();
1823 
1824  SDValue MaskLo, MaskHi;
1825  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1826  // Split Mask operand
1827  GetSplitVector(Mask, MaskLo, MaskHi);
1828  else
1829  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1830 
1831  EVT MemoryVT = MGT->getMemoryVT();
1832  EVT LoMemVT, HiMemVT;
1833  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1834 
1835  SDValue Src0Lo, Src0Hi;
1836  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1837  GetSplitVector(Src0, Src0Lo, Src0Hi);
1838  else
1839  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1840 
1841  SDValue IndexHi, IndexLo;
1842  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1843  GetSplitVector(Index, IndexLo, IndexHi);
1844  else
1845  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1846 
1848  getMachineMemOperand(MGT->getPointerInfo(),
1850  Alignment, MGT->getAAInfo(), MGT->getRanges());
1851 
1852  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
1853  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
1854  OpsLo, MMO);
1855 
1856  MMO = DAG.getMachineFunction().
1857  getMachineMemOperand(MGT->getPointerInfo(),
1859  Alignment, MGT->getAAInfo(),
1860  MGT->getRanges());
1861 
1862  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
1863  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
1864  OpsHi, MMO);
1865 
1866  // Build a factor node to remember that this load is independent of the
1867  // other one.
1868  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1869  Hi.getValue(1));
1870 
1871  // Legalize the chain result - switch anything that used the old chain to
1872  // use the new one.
1873  ReplaceValueWith(SDValue(MGT, 1), Ch);
1874 
1875  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
1876  Hi);
1877  ReplaceValueWith(SDValue(MGT, 0), Res);
1878  return SDValue();
1879 }
1880 
1881 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
1882  unsigned OpNo) {
1883  SDValue Ch = N->getChain();
1884  SDValue Ptr = N->getBasePtr();
1885  SDValue Mask = N->getMask();
1886  SDValue Data = N->getValue();
1887  EVT MemoryVT = N->getMemoryVT();
1888  unsigned Alignment = N->getOriginalAlignment();
1889  SDLoc DL(N);
1890 
1891  EVT LoMemVT, HiMemVT;
1892  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1893 
1894  SDValue DataLo, DataHi;
1895  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1896  // Split Data operand
1897  GetSplitVector(Data, DataLo, DataHi);
1898  else
1899  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1900 
1901  SDValue MaskLo, MaskHi;
1902  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1903  // Split Mask operand
1904  GetSplitVector(Mask, MaskLo, MaskHi);
1905  else
1906  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1907 
1908  // if Alignment is equal to the vector size,
1909  // take the half of it for the second part
1910  unsigned SecondHalfAlignment =
1911  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
1912  Alignment/2 : Alignment;
1913 
1914  SDValue Lo, Hi;
1916  getMachineMemOperand(N->getPointerInfo(),
1918  Alignment, N->getAAInfo(), N->getRanges());
1919 
1920  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
1921  N->isTruncatingStore(),
1922  N->isCompressingStore());
1923 
1924  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
1925  N->isCompressingStore());
1926  MMO = DAG.getMachineFunction().
1927  getMachineMemOperand(N->getPointerInfo(),
1929  SecondHalfAlignment, N->getAAInfo(), N->getRanges());
1930 
1931  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
1933 
1934  // Build a factor node to remember that this store is independent of the
1935  // other one.
1936  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1937 }
1938 
1939 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
1940  unsigned OpNo) {
1941  SDValue Ch = N->getChain();
1942  SDValue Ptr = N->getBasePtr();
1943  SDValue Mask = N->getMask();
1944  SDValue Index = N->getIndex();
1945  SDValue Scale = N->getScale();
1946  SDValue Data = N->getValue();
1947  EVT MemoryVT = N->getMemoryVT();
1948  unsigned Alignment = N->getOriginalAlignment();
1949  SDLoc DL(N);
1950 
1951  // Split all operands
1952  EVT LoMemVT, HiMemVT;
1953  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1954 
1955  SDValue DataLo, DataHi;
1956  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1957  // Split Data operand
1958  GetSplitVector(Data, DataLo, DataHi);
1959  else
1960  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1961 
1962  SDValue MaskLo, MaskHi;
1963  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1964  // Split Mask operand
1965  GetSplitVector(Mask, MaskLo, MaskHi);
1966  else
1967  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1968 
1969  SDValue IndexHi, IndexLo;
1970  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1971  GetSplitVector(Index, IndexLo, IndexHi);
1972  else
1973  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
1974 
1975  SDValue Lo;
1977  getMachineMemOperand(N->getPointerInfo(),
1979  Alignment, N->getAAInfo(), N->getRanges());
1980 
1981  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
1982  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
1983  DL, OpsLo, MMO);
1984 
1985  MMO = DAG.getMachineFunction().
1986  getMachineMemOperand(N->getPointerInfo(),
1988  Alignment, N->getAAInfo(), N->getRanges());
1989 
1990  // The order of the Scatter operation after split is well defined. The "Hi"
1991  // part comes after the "Lo". So these two operations should be chained one
1992  // after another.
1993  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
1994  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
1995  DL, OpsHi, MMO);
1996 }
1997 
1998 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
1999  assert(N->isUnindexed() && "Indexed store of vector?");
2000  assert(OpNo == 1 && "Can only split the stored value");
2001  SDLoc DL(N);
2002 
2003  bool isTruncating = N->isTruncatingStore();
2004  SDValue Ch = N->getChain();
2005  SDValue Ptr = N->getBasePtr();
2006  EVT MemoryVT = N->getMemoryVT();
2007  unsigned Alignment = N->getOriginalAlignment();
2008  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2009  AAMDNodes AAInfo = N->getAAInfo();
2010  SDValue Lo, Hi;
2011  GetSplitVector(N->getOperand(1), Lo, Hi);
2012 
2013  EVT LoMemVT, HiMemVT;
2014  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2015 
2016  // Scalarize if the split halves are not byte-sized.
2017  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2018  return TLI.scalarizeVectorStore(N, DAG);
2019 
2020  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2021 
2022  if (isTruncating)
2023  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2024  Alignment, MMOFlags, AAInfo);
2025  else
2026  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2027  AAInfo);
2028 
2029  // Increment the pointer to the other half.
2030  Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2031 
2032  if (isTruncating)
2033  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2034  N->getPointerInfo().getWithOffset(IncrementSize),
2035  HiMemVT, Alignment, MMOFlags, AAInfo);
2036  else
2037  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2038  N->getPointerInfo().getWithOffset(IncrementSize),
2039  Alignment, MMOFlags, AAInfo);
2040 
2041  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2042 }
2043 
2044 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2045  SDLoc DL(N);
2046 
2047  // The input operands all must have the same type, and we know the result
2048  // type is valid. Convert this to a buildvector which extracts all the
2049  // input elements.
2050  // TODO: If the input elements are power-two vectors, we could convert this to
2051  // a new CONCAT_VECTORS node with elements that are half-wide.
2053  EVT EltVT = N->getValueType(0).getVectorElementType();
2054  for (const SDValue &Op : N->op_values()) {
2055  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2056  i != e; ++i) {
2057  Elts.push_back(DAG.getNode(
2058  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2059  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2060  }
2061  }
2062 
2063  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2064 }
2065 
2066 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2067  // The result type is legal, but the input type is illegal. If splitting
2068  // ends up with the result type of each half still being legal, just
2069  // do that. If, however, that would result in an illegal result type,
2070  // we can try to get more clever with power-two vectors. Specifically,
2071  // split the input type, but also widen the result element size, then
2072  // concatenate the halves and truncate again. For example, consider a target
2073  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2074  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2075  // %inlo = v4i32 extract_subvector %in, 0
2076  // %inhi = v4i32 extract_subvector %in, 4
2077  // %lo16 = v4i16 trunc v4i32 %inlo
2078  // %hi16 = v4i16 trunc v4i32 %inhi
2079  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2080  // %res = v8i8 trunc v8i16 %in16
2081  //
2082  // Without this transform, the original truncate would end up being
2083  // scalarized, which is pretty much always a last resort.
2084  SDValue InVec = N->getOperand(0);
2085  EVT InVT = InVec->getValueType(0);
2086  EVT OutVT = N->getValueType(0);
2087  unsigned NumElements = OutVT.getVectorNumElements();
2088  bool IsFloat = OutVT.isFloatingPoint();
2089 
2090  // Widening should have already made sure this is a power-two vector
2091  // if we're trying to split it at all. assert() that's true, just in case.
2092  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2093 
2094  unsigned InElementSize = InVT.getScalarSizeInBits();
2095  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2096 
2097  // If the input elements are only 1/2 the width of the result elements,
2098  // just use the normal splitting. Our trick only work if there's room
2099  // to split more than once.
2100  if (InElementSize <= OutElementSize * 2)
2101  return SplitVecOp_UnaryOp(N);
2102  SDLoc DL(N);
2103 
2104  // Extract the halves of the input via extract_subvector.
2105  SDValue InLoVec, InHiVec;
2106  std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL);
2107  // Truncate them to 1/2 the element size.
2108  EVT HalfElementVT = IsFloat ?
2109  EVT::getFloatingPointVT(InElementSize/2) :
2110  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2111  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2112  NumElements/2);
2113  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2114  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2115  // Concatenate them to get the full intermediate truncation result.
2116  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2117  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2118  HalfHi);
2119  // Now finish up by truncating all the way down to the original result
2120  // type. This should normally be something that ends up being legal directly,
2121  // but in theory if a target has very wide vectors and an annoyingly
2122  // restricted set of legal types, this split can chain to build things up.
2123  return IsFloat
2124  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2125  DAG.getTargetConstant(
2126  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2127  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2128 }
2129 
2130 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2131  assert(N->getValueType(0).isVector() &&
2132  N->getOperand(0).getValueType().isVector() &&
2133  "Operand types must be vectors");
2134  // The result has a legal vector type, but the input needs splitting.
2135  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2136  SDLoc DL(N);
2137  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2138  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2139  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2140  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2141  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2142 
2143  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2144  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2145  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2146  return PromoteTargetBoolean(Con, N->getValueType(0));
2147 }
2148 
2149 
2150 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2151  // The result has a legal vector type, but the input needs splitting.
2152  EVT ResVT = N->getValueType(0);
2153  SDValue Lo, Hi;
2154  SDLoc DL(N);
2155  GetSplitVector(N->getOperand(0), Lo, Hi);
2156  EVT InVT = Lo.getValueType();
2157 
2158  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2159  InVT.getVectorNumElements());
2160 
2161  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2162  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2163 
2164  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2165 }
2166 
2167 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2168  // The result (and the first input) has a legal vector type, but the second
2169  // input needs splitting.
2170  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2171 }
2172 
2173 
2174 //===----------------------------------------------------------------------===//
2175 // Result Vector Widening
2176 //===----------------------------------------------------------------------===//
2177 
2178 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2179  DEBUG(dbgs() << "Widen node result " << ResNo << ": ";
2180  N->dump(&DAG);
2181  dbgs() << "\n");
2182 
2183  // See if the target wants to custom widen this node.
2184  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2185  return;
2186 
2187  SDValue Res = SDValue();
2188  switch (N->getOpcode()) {
2189  default:
2190 #ifndef NDEBUG
2191  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2192  N->dump(&DAG);
2193  dbgs() << "\n";
2194 #endif
2195  llvm_unreachable("Do not know how to widen the result of this operator!");
2196 
2197  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2198  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2199  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2200  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2201  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2202  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2203  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2204  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2205  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2206  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2207  case ISD::VSELECT:
2208  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2209  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2210  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2211  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2212  case ISD::VECTOR_SHUFFLE:
2213  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2214  break;
2215  case ISD::MLOAD:
2216  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2217  break;
2218  case ISD::MGATHER:
2219  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2220  break;
2221 
2222  case ISD::ADD:
2223  case ISD::AND:
2224  case ISD::MUL:
2225  case ISD::MULHS:
2226  case ISD::MULHU:
2227  case ISD::OR:
2228  case ISD::SUB:
2229  case ISD::XOR:
2230  case ISD::FMINNUM:
2231  case ISD::FMAXNUM:
2232  case ISD::FMINNAN:
2233  case ISD::FMAXNAN:
2234  case ISD::SMIN:
2235  case ISD::SMAX:
2236  case ISD::UMIN:
2237  case ISD::UMAX:
2238  Res = WidenVecRes_Binary(N);
2239  break;
2240 
2241  case ISD::FADD:
2242  case ISD::FMUL:
2243  case ISD::FPOW:
2244  case ISD::FSUB:
2245  case ISD::FDIV:
2246  case ISD::FREM:
2247  case ISD::SDIV:
2248  case ISD::UDIV:
2249  case ISD::SREM:
2250  case ISD::UREM:
2251  Res = WidenVecRes_BinaryCanTrap(N);
2252  break;
2253 
2254  case ISD::FCOPYSIGN:
2255  Res = WidenVecRes_FCOPYSIGN(N);
2256  break;
2257 
2258  case ISD::FPOWI:
2259  Res = WidenVecRes_POWI(N);
2260  break;
2261 
2262  case ISD::SHL:
2263  case ISD::SRA:
2264  case ISD::SRL:
2265  Res = WidenVecRes_Shift(N);
2266  break;
2267 
2271  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2272  break;
2273 
2274  case ISD::ANY_EXTEND:
2275  case ISD::FP_EXTEND:
2276  case ISD::FP_ROUND:
2277  case ISD::FP_TO_SINT:
2278  case ISD::FP_TO_UINT:
2279  case ISD::SIGN_EXTEND:
2280  case ISD::SINT_TO_FP:
2281  case ISD::TRUNCATE:
2282  case ISD::UINT_TO_FP:
2283  case ISD::ZERO_EXTEND:
2284  Res = WidenVecRes_Convert(N);
2285  break;
2286 
2287  case ISD::BITREVERSE:
2288  case ISD::BSWAP:
2289  case ISD::CTLZ:
2290  case ISD::CTPOP:
2291  case ISD::CTTZ:
2292  case ISD::FABS:
2293  case ISD::FCEIL:
2294  case ISD::FCOS:
2295  case ISD::FEXP:
2296  case ISD::FEXP2:
2297  case ISD::FFLOOR:
2298  case ISD::FLOG:
2299  case ISD::FLOG10:
2300  case ISD::FLOG2:
2301  case ISD::FNEARBYINT:
2302  case ISD::FNEG:
2303  case ISD::FRINT:
2304  case ISD::FROUND:
2305  case ISD::FSIN:
2306  case ISD::FSQRT:
2307  case ISD::FTRUNC:
2308  Res = WidenVecRes_Unary(N);
2309  break;
2310  case ISD::FMA:
2311  Res = WidenVecRes_Ternary(N);
2312  break;
2313  }
2314 
2315  // If Res is null, the sub-method took care of registering the result.
2316  if (Res.getNode())
2317  SetWidenedVector(SDValue(N, ResNo), Res);
2318 }
2319 
2320 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2321  // Ternary op widening.
2322  SDLoc dl(N);
2323  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2324  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2325  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2326  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2327  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2328 }
2329 
2330 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2331  // Binary op widening.
2332  SDLoc dl(N);
2333  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2334  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2335  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2336  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2337 }
2338 
2339 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2340  // Binary op widening for operations that can trap.
2341  unsigned Opcode = N->getOpcode();
2342  SDLoc dl(N);
2343  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2344  EVT WidenEltVT = WidenVT.getVectorElementType();
2345  EVT VT = WidenVT;
2346  unsigned NumElts = VT.getVectorNumElements();
2347  const SDNodeFlags Flags = N->getFlags();
2348  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2349  NumElts = NumElts / 2;
2350  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2351  }
2352 
2353  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2354  // Operation doesn't trap so just widen as normal.
2355  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2356  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2357  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2358  }
2359 
2360  // No legal vector version so unroll the vector operation and then widen.
2361  if (NumElts == 1)
2362  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2363 
2364  // Since the operation can trap, apply operation on the original vector.
2365  EVT MaxVT = VT;
2366  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2367  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2368  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2369 
2370  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2371  unsigned ConcatEnd = 0; // Current ConcatOps index.
2372  int Idx = 0; // Current Idx into input vectors.
2373 
2374  // NumElts := greatest legal vector size (at most WidenVT)
2375  // while (orig. vector has unhandled elements) {
2376  // take munches of size NumElts from the beginning and add to ConcatOps
2377  // NumElts := next smaller supported vector size or 1
2378  // }
2379  while (CurNumElts != 0) {
2380  while (CurNumElts >= NumElts) {
2381  SDValue EOp1 = DAG.getNode(
2382  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2383  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2384  SDValue EOp2 = DAG.getNode(
2385  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2386  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2387  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2388  Idx += NumElts;
2389  CurNumElts -= NumElts;
2390  }
2391  do {
2392  NumElts = NumElts / 2;
2393  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2394  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2395 
2396  if (NumElts == 1) {
2397  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2398  SDValue EOp1 = DAG.getNode(
2399  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2400  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2401  SDValue EOp2 = DAG.getNode(
2402  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2403  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2404  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2405  EOp1, EOp2, Flags);
2406  }
2407  CurNumElts = 0;
2408  }
2409  }
2410 
2411  // Check to see if we have a single operation with the widen type.
2412  if (ConcatEnd == 1) {
2413  VT = ConcatOps[0].getValueType();
2414  if (VT == WidenVT)
2415  return ConcatOps[0];
2416  }
2417 
2418  // while (Some element of ConcatOps is not of type MaxVT) {
2419  // From the end of ConcatOps, collect elements of the same type and put
2420  // them into an op of the next larger supported type
2421  // }
2422  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2423  Idx = ConcatEnd - 1;
2424  VT = ConcatOps[Idx--].getValueType();
2425  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2426  Idx--;
2427 
2428  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2429  EVT NextVT;
2430  do {
2431  NextSize *= 2;
2432  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2433  } while (!TLI.isTypeLegal(NextVT));
2434 
2435  if (!VT.isVector()) {
2436  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2437  SDValue VecOp = DAG.getUNDEF(NextVT);
2438  unsigned NumToInsert = ConcatEnd - Idx - 1;
2439  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2440  VecOp = DAG.getNode(
2441  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2442  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2443  }
2444  ConcatOps[Idx+1] = VecOp;
2445  ConcatEnd = Idx + 2;
2446  } else {
2447  // Vector type, create a CONCAT_VECTORS of type NextVT
2448  SDValue undefVec = DAG.getUNDEF(VT);
2449  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2450  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2451  unsigned RealVals = ConcatEnd - Idx - 1;
2452  unsigned SubConcatEnd = 0;
2453  unsigned SubConcatIdx = Idx + 1;
2454  while (SubConcatEnd < RealVals)
2455  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2456  while (SubConcatEnd < OpsToConcat)
2457  SubConcatOps[SubConcatEnd++] = undefVec;
2458  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2459  NextVT, SubConcatOps);
2460  ConcatEnd = SubConcatIdx + 1;
2461  }
2462  }
2463 
2464  // Check to see if we have a single operation with the widen type.
2465  if (ConcatEnd == 1) {
2466  VT = ConcatOps[0].getValueType();
2467  if (VT == WidenVT)
2468  return ConcatOps[0];
2469  }
2470 
2471  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2472  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2473  if (NumOps != ConcatEnd ) {
2474  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2475  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2476  ConcatOps[j] = UndefVal;
2477  }
2478  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2479  makeArrayRef(ConcatOps.data(), NumOps));
2480 }
2481 
2482 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
2483  SDValue InOp = N->getOperand(0);
2484  SDLoc DL(N);
2485 
2486  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2487  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2488 
2489  EVT InVT = InOp.getValueType();
2490  EVT InEltVT = InVT.getVectorElementType();
2491  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2492 
2493  unsigned Opcode = N->getOpcode();
2494  unsigned InVTNumElts = InVT.getVectorNumElements();
2495  const SDNodeFlags Flags = N->getFlags();
2496  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2497  InOp = GetWidenedVector(N->getOperand(0));
2498  InVT = InOp.getValueType();
2499  InVTNumElts = InVT.getVectorNumElements();
2500  if (InVTNumElts == WidenNumElts) {
2501  if (N->getNumOperands() == 1)
2502  return DAG.getNode(Opcode, DL, WidenVT, InOp);
2503  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
2504  }
2505  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
2506  // If both input and result vector types are of same width, extend
2507  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
2508  // accepts fewer elements in the result than in the input.
2509  if (Opcode == ISD::SIGN_EXTEND)
2510  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2511  if (Opcode == ISD::ZERO_EXTEND)
2512  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2513  }
2514  }
2515 
2516  if (TLI.isTypeLegal(InWidenVT)) {
2517  // Because the result and the input are different vector types, widening
2518  // the result could create a legal type but widening the input might make
2519  // it an illegal type that might lead to repeatedly splitting the input
2520  // and then widening it. To avoid this, we widen the input only if
2521  // it results in a legal type.
2522  if (WidenNumElts % InVTNumElts == 0) {
2523  // Widen the input and call convert on the widened input vector.
2524  unsigned NumConcat = WidenNumElts/InVTNumElts;
2525  SmallVector<SDValue, 16> Ops(NumConcat);
2526  Ops[0] = InOp;
2527  SDValue UndefVal = DAG.getUNDEF(InVT);
2528  for (unsigned i = 1; i != NumConcat; ++i)
2529  Ops[i] = UndefVal;
2530  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
2531  if (N->getNumOperands() == 1)
2532  return DAG.getNode(Opcode, DL, WidenVT, InVec);
2533  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
2534  }
2535 
2536  if (InVTNumElts % WidenNumElts == 0) {
2537  SDValue InVal = DAG.getNode(
2538  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
2539  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2540  // Extract the input and convert the shorten input vector.
2541  if (N->getNumOperands() == 1)
2542  return DAG.getNode(Opcode, DL, WidenVT, InVal);
2543  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
2544  }
2545  }
2546 
2547  // Otherwise unroll into some nasty scalar code and rebuild the vector.
2548  SmallVector<SDValue, 16> Ops(WidenNumElts);
2549  EVT EltVT = WidenVT.getVectorElementType();
2550  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2551  unsigned i;
2552  for (i=0; i < MinElts; ++i) {
2553  SDValue Val = DAG.getNode(
2554  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
2555  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2556  if (N->getNumOperands() == 1)
2557  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
2558  else
2559  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
2560  }
2561 
2562  SDValue UndefVal = DAG.getUNDEF(EltVT);
2563  for (; i < WidenNumElts; ++i)
2564  Ops[i] = UndefVal;
2565 
2566  return DAG.getBuildVector(WidenVT, DL, Ops);
2567 }
2568 
2569 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
2570  unsigned Opcode = N->getOpcode();
2571  SDValue InOp = N->getOperand(0);
2572  SDLoc DL(N);
2573 
2574  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2575  EVT WidenSVT = WidenVT.getVectorElementType();
2576  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2577 
2578  EVT InVT = InOp.getValueType();
2579  EVT InSVT = InVT.getVectorElementType();
2580  unsigned InVTNumElts = InVT.getVectorNumElements();
2581 
2582  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2583  InOp = GetWidenedVector(InOp);
2584  InVT = InOp.getValueType();
2585  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
2586  switch (Opcode) {
2588  return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
2590  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2592  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2593  }
2594  }
2595  }
2596 
2597  // Unroll, extend the scalars and rebuild the vector.
2599  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
2600  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
2601  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2602  switch (Opcode) {
2604  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
2605  break;
2607  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
2608  break;
2610  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
2611  break;
2612  default:
2613  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
2614  }
2615  Ops.push_back(Val);
2616  }
2617 
2618  while (Ops.size() != WidenNumElts)
2619  Ops.push_back(DAG.getUNDEF(WidenSVT));
2620 
2621  return DAG.getBuildVector(WidenVT, DL, Ops);
2622 }
2623 
2624 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
2625  // If this is an FCOPYSIGN with same input types, we can treat it as a
2626  // normal (can trap) binary op.
2627  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
2628  return WidenVecRes_BinaryCanTrap(N);
2629 
2630  // If the types are different, fall back to unrolling.
2631  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2632  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2633 }
2634 
2635 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
2636  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2637  SDValue InOp = GetWidenedVector(N->getOperand(0));
2638  SDValue ShOp = N->getOperand(1);
2639  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2640 }
2641 
2642 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
2643  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2644  SDValue InOp = GetWidenedVector(N->getOperand(0));
2645  SDValue ShOp = N->getOperand(1);
2646 
2647  EVT ShVT = ShOp.getValueType();
2648  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
2649  ShOp = GetWidenedVector(ShOp);
2650  ShVT = ShOp.getValueType();
2651  }
2652  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
2653  ShVT.getVectorElementType(),
2654  WidenVT.getVectorNumElements());
2655  if (ShVT != ShWidenVT)
2656  ShOp = ModifyToType(ShOp, ShWidenVT);
2657 
2658  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2659 }
2660 
2661 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
2662  // Unary op widening.
2663  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2664  SDValue InOp = GetWidenedVector(N->getOperand(0));
2665  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
2666 }
2667 
2668 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
2669  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2670  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
2671  cast<VTSDNode>(N->getOperand(1))->getVT()
2672  .getVectorElementType(),
2673  WidenVT.getVectorNumElements());
2674  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
2675  return DAG.getNode(N->getOpcode(), SDLoc(N),
2676  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
2677 }
2678 
2679 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
2680  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2681  return GetWidenedVector(WidenVec);
2682 }
2683 
2684 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
2685  SDValue InOp = N->getOperand(0);
2686  EVT InVT = InOp.getValueType();
2687  EVT VT = N->getValueType(0);
2688  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2689  SDLoc dl(N);
2690 
2691  switch (getTypeAction(InVT)) {
2693  break;
2695  // If the incoming type is a vector that is being promoted, then
2696  // we know that the elements are arranged differently and that we
2697  // must perform the conversion using a stack slot.
2698  if (InVT.isVector())
2699  break;
2700 
2701  // If the InOp is promoted to the same size, convert it. Otherwise,
2702  // fall out of the switch and widen the promoted input.
2703  InOp = GetPromotedInteger(InOp);
2704  InVT = InOp.getValueType();
2705  if (WidenVT.bitsEq(InVT))
2706  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2707  break;
2714  break;
2716  // If the InOp is widened to the same size, convert it. Otherwise, fall
2717  // out of the switch and widen the widened input.
2718  InOp = GetWidenedVector(InOp);
2719  InVT = InOp.getValueType();
2720  if (WidenVT.bitsEq(InVT))
2721  // The input widens to the same size. Convert to the widen value.
2722  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2723  break;
2724  }
2725 
2726  unsigned WidenSize = WidenVT.getSizeInBits();
2727  unsigned InSize = InVT.getSizeInBits();
2728  // x86mmx is not an acceptable vector element type, so don't try.
2729  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
2730  // Determine new input vector type. The new input vector type will use
2731  // the same element type (if its a vector) or use the input type as a
2732  // vector. It is the same size as the type to widen to.
2733  EVT NewInVT;
2734  unsigned NewNumElts = WidenSize / InSize;
2735  if (InVT.isVector()) {
2736  EVT InEltVT = InVT.getVectorElementType();
2737  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
2738  WidenSize / InEltVT.getSizeInBits());
2739  } else {
2740  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
2741  }
2742 
2743  if (TLI.isTypeLegal(NewInVT)) {
2744  // Because the result and the input are different vector types, widening
2745  // the result could create a legal type but widening the input might make
2746  // it an illegal type that might lead to repeatedly splitting the input
2747  // and then widening it. To avoid this, we widen the input only if
2748  // it results in a legal type.
2749  SmallVector<SDValue, 16> Ops(NewNumElts);
2750  SDValue UndefVal = DAG.getUNDEF(InVT);
2751  Ops[0] = InOp;
2752  for (unsigned i = 1; i < NewNumElts; ++i)
2753  Ops[i] = UndefVal;
2754 
2755  SDValue NewVec;
2756  if (InVT.isVector())
2757  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
2758  else
2759  NewVec = DAG.getBuildVector(NewInVT, dl, Ops);
2760  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
2761  }
2762  }
2763 
2764  return CreateStackStoreLoad(InOp, WidenVT);
2765 }
2766 
2767 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
2768  SDLoc dl(N);
2769  // Build a vector with undefined for the new nodes.
2770  EVT VT = N->getValueType(0);
2771 
2772  // Integer BUILD_VECTOR operands may be larger than the node's vector element
2773  // type. The UNDEFs need to have the same type as the existing operands.
2774  EVT EltVT = N->getOperand(0).getValueType();
2775  unsigned NumElts = VT.getVectorNumElements();
2776 
2777  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2778  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2779 
2780  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
2781  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
2782  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
2783 
2784  return DAG.getBuildVector(WidenVT, dl, NewOps);
2785 }
2786 
2787 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
2788  EVT InVT = N->getOperand(0).getValueType();
2789  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2790  SDLoc dl(N);
2791  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2792  unsigned NumInElts = InVT.getVectorNumElements();
2793  unsigned NumOperands = N->getNumOperands();
2794 
2795  bool InputWidened = false; // Indicates we need to widen the input.
2796  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
2797  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
2798  // Add undef vectors to widen to correct length.
2799  unsigned NumConcat = WidenVT.getVectorNumElements() /
2800  InVT.getVectorNumElements();
2801  SDValue UndefVal = DAG.getUNDEF(InVT);
2802  SmallVector<SDValue, 16> Ops(NumConcat);
2803  for (unsigned i=0; i < NumOperands; ++i)
2804  Ops[i] = N->getOperand(i);
2805  for (unsigned i = NumOperands; i != NumConcat; ++i)
2806  Ops[i] = UndefVal;
2807  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
2808  }
2809  } else {
2810  InputWidened = true;
2811  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
2812  // The inputs and the result are widen to the same value.
2813  unsigned i;
2814  for (i=1; i < NumOperands; ++i)
2815  if (!N->getOperand(i).isUndef())
2816  break;
2817 
2818  if (i == NumOperands)
2819  // Everything but the first operand is an UNDEF so just return the
2820  // widened first operand.
2821  return GetWidenedVector(N->getOperand(0));
2822 
2823  if (NumOperands == 2) {
2824  // Replace concat of two operands with a shuffle.
2825  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
2826  for (unsigned i = 0; i < NumInElts; ++i) {
2827  MaskOps[i] = i;
2828  MaskOps[i + NumInElts] = i + WidenNumElts;
2829  }
2830  return DAG.getVectorShuffle(WidenVT, dl,
2831  GetWidenedVector(N->getOperand(0)),
2832  GetWidenedVector(N->getOperand(1)),
2833  MaskOps);
2834  }
2835  }
2836  }
2837 
2838  // Fall back to use extracts and build vector.
2839  EVT EltVT = WidenVT.getVectorElementType();
2840  SmallVector<SDValue, 16> Ops(WidenNumElts);
2841  unsigned Idx = 0;
2842  for (unsigned i=0; i < NumOperands; ++i) {
2843  SDValue InOp = N->getOperand(i);
2844  if (InputWidened)
2845  InOp = GetWidenedVector(InOp);
2846  for (unsigned j=0; j < NumInElts; ++j)
2847  Ops[Idx++] = DAG.getNode(
2848  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2849  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2850  }
2851  SDValue UndefVal = DAG.getUNDEF(EltVT);
2852  for (; Idx < WidenNumElts; ++Idx)
2853  Ops[Idx] = UndefVal;
2854  return DAG.getBuildVector(WidenVT, dl, Ops);
2855 }
2856 
2857 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
2858  EVT VT = N->getValueType(0);
2859  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2860  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2861  SDValue InOp = N->getOperand(0);
2862  SDValue Idx = N->getOperand(1);
2863  SDLoc dl(N);
2864 
2865  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2866  InOp = GetWidenedVector(InOp);
2867 
2868  EVT InVT = InOp.getValueType();
2869 
2870  // Check if we can just return the input vector after widening.
2871  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2872  if (IdxVal == 0 && InVT == WidenVT)
2873  return InOp;
2874 
2875  // Check if we can extract from the vector.
2876  unsigned InNumElts = InVT.getVectorNumElements();
2877  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2878  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
2879 
2880  // We could try widening the input to the right length but for now, extract
2881  // the original elements, fill the rest with undefs and build a vector.
2882  SmallVector<SDValue, 16> Ops(WidenNumElts);
2883  EVT EltVT = VT.getVectorElementType();
2884  unsigned NumElts = VT.getVectorNumElements();
2885  unsigned i;
2886  for (i=0; i < NumElts; ++i)
2887  Ops[i] =
2888  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2889  DAG.getConstant(IdxVal + i, dl,
2890  TLI.getVectorIdxTy(DAG.getDataLayout())));
2891 
2892  SDValue UndefVal = DAG.getUNDEF(EltVT);
2893  for (; i < WidenNumElts; ++i)
2894  Ops[i] = UndefVal;
2895  return DAG.getBuildVector(WidenVT, dl, Ops);
2896 }
2897 
2898 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
2899  SDValue InOp = GetWidenedVector(N->getOperand(0));
2900  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
2901  InOp.getValueType(), InOp,
2902  N->getOperand(1), N->getOperand(2));
2903 }
2904 
2905 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
2906  LoadSDNode *LD = cast<LoadSDNode>(N);
2907  ISD::LoadExtType ExtType = LD->getExtensionType();
2908 
2909  SDValue Result;
2910  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
2911  if (ExtType != ISD::NON_EXTLOAD)
2912  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2913  else
2914  Result = GenWidenVectorLoads(LdChain, LD);
2915 
2916  // If we generate a single load, we can use that for the chain. Otherwise,
2917  // build a factor node to remember the multiple loads are independent and
2918  // chain to that.
2919  SDValue NewChain;
2920  if (LdChain.size() == 1)
2921  NewChain = LdChain[0];
2922  else
2923  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
2924 
2925  // Modified the chain - switch anything that used the old chain to use
2926  // the new one.
2927  ReplaceValueWith(SDValue(N, 1), NewChain);
2928 
2929  return Result;
2930 }
2931 
2932 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
2933 
2934  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
2935  SDValue Mask = N->getMask();
2936  EVT MaskVT = Mask.getValueType();
2937  SDValue Src0 = GetWidenedVector(N->getSrc0());
2938  ISD::LoadExtType ExtType = N->getExtensionType();
2939  SDLoc dl(N);
2940 
2941  // The mask should be widened as well
2942  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
2943  MaskVT.getVectorElementType(),
2944  WidenVT.getVectorNumElements());
2945  Mask = ModifyToType(Mask, WideMaskVT, true);
2946 
2947  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
2948  Mask, Src0, N->getMemoryVT(),
2949  N->getMemOperand(), ExtType,
2950  N->isExpandingLoad());
2951  // Legalize the chain result - switch anything that used the old chain to
2952  // use the new one.
2953  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2954  return Res;
2955 }
2956 
2957 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
2958 
2959  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2960  SDValue Mask = N->getMask();
2961  EVT MaskVT = Mask.getValueType();
2962  SDValue Src0 = GetWidenedVector(N->getValue());
2963  SDValue Scale = N->getScale();
2964  unsigned NumElts = WideVT.getVectorNumElements();
2965  SDLoc dl(N);
2966 
2967  // The mask should be widened as well
2968  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
2969  MaskVT.getVectorElementType(),
2970  WideVT.getVectorNumElements());
2971  Mask = ModifyToType(Mask, WideMaskVT, true);
2972 
2973  // Widen the Index operand
2974  SDValue Index = N->getIndex();
2975  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
2976  Index.getValueType().getScalarType(),
2977  NumElts);
2978  Index = ModifyToType(Index, WideIndexVT);
2979  SDValue Ops[] = { N->getChain(), Src0, Mask, N->getBasePtr(), Index, Scale };
2980  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
2981  N->getMemoryVT(), dl, Ops,
2982  N->getMemOperand());
2983 
2984  // Legalize the chain result - switch anything that used the old chain to
2985  // use the new one.
2986  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2987  return Res;
2988 }
2989 
2990 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
2991  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2992  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
2993  WidenVT, N->getOperand(0));
2994 }
2995 
2996 // Return true if this is a node that could have two SETCCs as operands.
2997 static inline bool isLogicalMaskOp(unsigned Opcode) {
2998  switch (Opcode) {
2999  case ISD::AND:
3000  case ISD::OR:
3001  case ISD::XOR:
3002  return true;
3003  }
3004  return false;
3005 }
3006 
3007 // This is used just for the assert in convertMask(). Check that this either
3008 // a SETCC or a previously handled SETCC by convertMask().
3009 #ifndef NDEBUG
3010 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3011  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3012  N = N.getOperand(0);
3013  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3014  for (unsigned i = 1; i < N->getNumOperands(); ++i)
3015  if (!N->getOperand(i)->isUndef())
3016  return false;
3017  N = N.getOperand(0);
3018  }
3019 
3020  if (N.getOpcode() == ISD::TRUNCATE)
3021  N = N.getOperand(0);
3022  else if (N.getOpcode() == ISD::SIGN_EXTEND)
3023  N = N.getOperand(0);
3024 
3025  if (isLogicalMaskOp(N.getOpcode()))
3026  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3028 
3029  return (N.getOpcode() == ISD::SETCC ||
3031 }
3032 #endif
3033 
3034 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3035 // to ToMaskVT if needed with vector extension or truncation.
3036 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3037  EVT ToMaskVT) {
3038  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3039  // FIXME: This code seems to be too restrictive, we might consider
3040  // generalizing it or dropping it.
3041  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3042 
3043  // Make a new Mask node, with a legal result VT.
3045  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3046  Ops.push_back(InMask->getOperand(i));
3047  SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3048 
3049  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3050  // extend or truncate is needed.
3051  LLVMContext &Ctx = *DAG.getContext();
3052  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3053  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3054  if (MaskScalarBits < ToMaskScalBits) {
3055  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3056  MaskVT.getVectorNumElements());
3057  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3058  } else if (MaskScalarBits > ToMaskScalBits) {
3059  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3060  MaskVT.getVectorNumElements());
3061  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3062  }
3063 
3065  ToMaskVT.getScalarSizeInBits() &&
3066  "Mask should have the right element size by now.");
3067 
3068  // Adjust Mask to the right number of elements.
3069  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3070  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3071  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3072  SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3073  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3074  ZeroIdx);
3075  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3076  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3077  EVT SubVT = Mask->getValueType(0);
3078  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3079  SubOps[0] = Mask;
3080  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3081  }
3082 
3083  assert((Mask->getValueType(0) == ToMaskVT) &&
3084  "A mask of ToMaskVT should have been produced by now.");
3085 
3086  return Mask;
3087 }
3088 
3089 // Get the target mask VT, and widen if needed.
3090 EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
3091  assert(SetCC->getOpcode() == ISD::SETCC);
3092  LLVMContext &Ctx = *DAG.getContext();
3093  EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
3094  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3095  MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
3096  return MaskVT;
3097 }
3098 
3099 // This method tries to handle VSELECT and its mask by legalizing operands
3100 // (which may require widening) and if needed adjusting the mask vector type
3101 // to match that of the VSELECT. Without it, many cases end up with
3102 // scalarization of the SETCC, with many unnecessary instructions.
3103 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3104  LLVMContext &Ctx = *DAG.getContext();
3105  SDValue Cond = N->getOperand(0);
3106 
3107  if (N->getOpcode() != ISD::VSELECT)
3108  return SDValue();
3109 
3110  if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3111  return SDValue();
3112 
3113  // If this is a splitted VSELECT that was previously already handled, do
3114  // nothing.
3115  EVT CondVT = Cond->getValueType(0);
3116  if (CondVT.getScalarSizeInBits() != 1)
3117  return SDValue();
3118 
3119  EVT VSelVT = N->getValueType(0);
3120  // Only handle vector types which are a power of 2.
3121  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3122  return SDValue();
3123 
3124  // Don't touch if this will be scalarized.
3125  EVT FinalVT = VSelVT;
3126  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3127  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3128 
3129  if (FinalVT.getVectorNumElements() == 1)
3130  return SDValue();
3131 
3132  // If there is support for an i1 vector mask, don't touch.
3133  if (Cond.getOpcode() == ISD::SETCC) {
3134  EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3135  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3136  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3137  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3138  if (SetCCResVT.getScalarSizeInBits() == 1)
3139  return SDValue();
3140  } else if (CondVT.getScalarType() == MVT::i1) {
3141  // If there is support for an i1 vector mask (or only scalar i1 conditions),
3142  // don't touch.
3143  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3144  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3145 
3146  if (CondVT.getScalarType() == MVT::i1)
3147  return SDValue();
3148  }
3149 
3150  // Get the VT and operands for VSELECT, and widen if needed.
3151  SDValue VSelOp1 = N->getOperand(1);
3152  SDValue VSelOp2 = N->getOperand(2);
3153  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3154  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3155  VSelOp1 = GetWidenedVector(VSelOp1);
3156  VSelOp2 = GetWidenedVector(VSelOp2);
3157  }
3158 
3159  // The mask of the VSELECT should have integer elements.
3160  EVT ToMaskVT = VSelVT;
3161  if (!ToMaskVT.getScalarType().isInteger())
3162  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3163 
3164  SDValue Mask;
3165  if (Cond->getOpcode() == ISD::SETCC) {
3166  EVT MaskVT = getSETCCWidenedResultTy(Cond);
3167  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3168  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3169  Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3170  Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3171  // Cond is (AND/OR/XOR (SETCC, SETCC))
3172  SDValue SETCC0 = Cond->getOperand(0);
3173  SDValue SETCC1 = Cond->getOperand(1);
3174  EVT VT0 = getSETCCWidenedResultTy(SETCC0);
3175  EVT VT1 = getSETCCWidenedResultTy(SETCC1);
3176  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3177  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3178  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3179  EVT MaskVT;
3180  // If the two SETCCs have different VTs, either extend/truncate one of
3181  // them to the other "towards" ToMaskVT, or truncate one and extend the
3182  // other to ToMaskVT.
3183  if (ScalarBits0 != ScalarBits1) {
3184  EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3185  EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3186  if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3187  MaskVT = WideVT;
3188  else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3189  MaskVT = NarrowVT;
3190  else
3191  MaskVT = ToMaskVT;
3192  } else
3193  // If the two SETCCs have the same VT, don't change it.
3194  MaskVT = VT0;
3195 
3196  // Make new SETCCs and logical nodes.
3197  SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3198  SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3199  Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3200 
3201  // Convert the logical op for VSELECT if needed.
3202  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3203  } else
3204  return SDValue();
3205 
3206  return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3207 }
3208 
3209 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3210  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3211  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3212 
3213  SDValue Cond1 = N->getOperand(0);
3214  EVT CondVT = Cond1.getValueType();
3215  if (CondVT.isVector()) {
3216  if (SDValue Res = WidenVSELECTAndMask(N))
3217  return Res;
3218 
3219  EVT CondEltVT = CondVT.getVectorElementType();
3220  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3221  CondEltVT, WidenNumElts);
3222  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3223  Cond1 = GetWidenedVector(Cond1);
3224 
3225  // If we have to split the condition there is no point in widening the
3226  // select. This would result in an cycle of widening the select ->
3227  // widening the condition operand -> splitting the condition operand ->
3228  // splitting the select -> widening the select. Instead split this select
3229  // further and widen the resulting type.
3230  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3231  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3232  SDValue Res = ModifyToType(SplitSelect, WidenVT);
3233  return Res;
3234  }
3235 
3236  if (Cond1.getValueType() != CondWidenVT)
3237  Cond1 = ModifyToType(Cond1, CondWidenVT);
3238  }
3239 
3240  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3241  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3242  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3243  return DAG.getNode(N->getOpcode(), SDLoc(N),
3244  WidenVT, Cond1, InOp1, InOp2);
3245 }
3246 
3247 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3248  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3249  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3250  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3251  InOp1.getValueType(), N->getOperand(0),
3252  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3253 }
3254 
3255 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3256  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3257  return DAG.getUNDEF(WidenVT);
3258 }
3259 
3260 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3261  EVT VT = N->getValueType(0);
3262  SDLoc dl(N);
3263 
3264  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3265  unsigned NumElts = VT.getVectorNumElements();
3266  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3267 
3268  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3269  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3270 
3271  // Adjust mask based on new input vector length.
3272  SmallVector<int, 16> NewMask;
3273  for (unsigned i = 0; i != NumElts; ++i) {
3274  int Idx = N->getMaskElt(i);
3275  if (Idx < (int)NumElts)
3276  NewMask.push_back(Idx);
3277  else
3278  NewMask.push_back(Idx - NumElts + WidenNumElts);
3279  }
3280  for (unsigned i = NumElts; i != WidenNumElts; ++i)
3281  NewMask.push_back(-1);
3282  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
3283 }
3284 
3285 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
3286  assert(N->getValueType(0).isVector() &&
3287  N->getOperand(0).getValueType().isVector() &&
3288  "Operands must be vectors");
3289  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3290  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3291 
3292  SDValue InOp1 = N->getOperand(0);
3293  EVT InVT = InOp1.getValueType();
3294  assert(InVT.isVector() && "can not widen non-vector type");
3295  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
3296  InVT.getVectorElementType(), WidenNumElts);
3297 
3298  // The input and output types often differ here, and it could be that while
3299  // we'd prefer to widen the result type, the input operands have been split.
3300  // In this case, we also need to split the result of this node as well.
3301  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
3302  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
3303  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
3304  return Res;
3305  }
3306 
3307  InOp1 = GetWidenedVector(InOp1);
3308  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3309 
3310  // Assume that the input and output will be widen appropriately. If not,
3311  // we will have to unroll it at some point.
3312  assert(InOp1.getValueType() == WidenInVT &&
3313  InOp2.getValueType() == WidenInVT &&
3314  "Input not widened to expected type!");
3315  (void)WidenInVT;
3316  return DAG.getNode(ISD::SETCC, SDLoc(N),
3317  WidenVT, InOp1, InOp2, N->getOperand(2));
3318 }
3319 
3320 
3321 //===----------------------------------------------------------------------===//
3322 // Widen Vector Operand
3323 //===----------------------------------------------------------------------===//
3324 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
3325  DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
3326  N->dump(&DAG);
3327  dbgs() << "\n");
3328  SDValue Res = SDValue();
3329 
3330  // See if the target wants to custom widen this node.
3331  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3332  return false;
3333 
3334  switch (N->getOpcode()) {
3335  default:
3336 #ifndef NDEBUG
3337  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
3338  N->dump(&DAG);
3339  dbgs() << "\n";
3340 #endif
3341  llvm_unreachable("Do not know how to widen this operator's operand!");
3342 
3343  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
3344  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
3345  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
3346  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
3347  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
3348  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
3349  case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
3350  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
3351  case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
3352 
3353  case ISD::ANY_EXTEND:
3354  case ISD::SIGN_EXTEND:
3355  case ISD::ZERO_EXTEND:
3356  Res = WidenVecOp_EXTEND(N);
3357  break;
3358 
3359  case ISD::FP_EXTEND:
3360  case ISD::FP_TO_SINT:
3361  case ISD::FP_TO_UINT:
3362  case ISD::SINT_TO_FP:
3363  case ISD::UINT_TO_FP:
3364  case ISD::TRUNCATE:
3365  Res = WidenVecOp_Convert(N);
3366  break;
3367  }
3368 
3369  // If Res is null, the sub-method took care of registering the result.
3370  if (!Res.getNode()) return false;
3371 
3372  // If the result is N, the sub-method updated N in place. Tell the legalizer
3373  // core about this.
3374  if (Res.getNode() == N)
3375  return true;
3376 
3377 
3378  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3379  "Invalid operand expansion");
3380 
3381  ReplaceValueWith(SDValue(N, 0), Res);
3382  return false;
3383 }
3384 
3385 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
3386  SDLoc DL(N);
3387  EVT VT = N->getValueType(0);
3388 
3389  SDValue InOp = N->getOperand(0);
3390  assert(getTypeAction(InOp.getValueType()) ==
3392  "Unexpected type action");
3393  InOp = GetWidenedVector(InOp);
3396  "Input wasn't widened!");
3397 
3398  // We may need to further widen the operand until it has the same total
3399  // vector size as the result.
3400  EVT InVT = InOp.getValueType();
3401  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
3402  EVT InEltVT = InVT.getVectorElementType();
3403  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
3404  EVT FixedVT = (MVT::SimpleValueType)i;
3405  EVT FixedEltVT = FixedVT.getVectorElementType();
3406  if (TLI.isTypeLegal(FixedVT) &&
3407  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
3408  FixedEltVT == InEltVT) {
3409  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
3410  "Not enough elements in the fixed type for the operand!");
3411  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
3412  "We can't have the same type as we started with!");
3413  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
3414  InOp = DAG.getNode(
3415  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
3416  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3417  else
3418  InOp = DAG.getNode(
3419  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
3420  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3421  break;
3422  }
3423  }
3424  InVT = InOp.getValueType();
3425  if (InVT.getSizeInBits() != VT.getSizeInBits())
3426  // We couldn't find a legal vector type that was a widening of the input
3427  // and could be extended in-register to the result type, so we have to
3428  // scalarize.
3429  return WidenVecOp_Convert(N);
3430  }
3431 
3432  // Use special DAG nodes to represent the operation of extending the
3433  // low lanes.
3434  switch (N->getOpcode()) {
3435  default:
3436  llvm_unreachable("Extend legalization on on extend operation!");
3437  case ISD::ANY_EXTEND:
3438  return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
3439  case ISD::SIGN_EXTEND:
3440  return DAG.getSignExtendVectorInReg(InOp, DL, VT);
3441  case ISD::ZERO_EXTEND:
3442  return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
3443  }
3444 }
3445 
3446 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
3447  // The result (and first input) is legal, but the second input is illegal.
3448  // We can't do much to fix that, so just unroll and let the extracts off of
3449  // the second input be widened as needed later.
3450  return DAG.UnrollVectorOp(N);
3451 }
3452 
3453 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
3454  // Since the result is legal and the input is illegal.
3455  EVT VT = N->getValueType(0);
3456  EVT EltVT = VT.getVectorElementType();
3457  SDLoc dl(N);
3458  unsigned NumElts = VT.getVectorNumElements();
3459  SDValue InOp = N->getOperand(0);
3460  assert(getTypeAction(InOp.getValueType()) ==
3462  "Unexpected type action");
3463  InOp = GetWidenedVector(InOp);
3464  EVT InVT = InOp.getValueType();
3465  unsigned Opcode = N->getOpcode();
3466 
3467  // See if a widened result type would be legal, if so widen the node.
3468  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3469  InVT.getVectorNumElements());
3470  if (TLI.isTypeLegal(WideVT)) {
3471  SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp);
3472  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
3473  DAG.getIntPtrConstant(0, dl));
3474  }
3475 
3476  EVT InEltVT = InVT.getVectorElementType();
3477 
3478  // Unroll the convert into some scalar code and create a nasty build vector.
3479  SmallVector<SDValue, 16> Ops(NumElts);
3480  for (unsigned i=0; i < NumElts; ++i)
3481  Ops[i] = DAG.getNode(
3482  Opcode, dl, EltVT,
3483  DAG.getNode(
3484  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
3485  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3486 
3487  return DAG.getBuildVector(VT, dl, Ops);
3488 }
3489 
3490 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
3491  EVT VT = N->getValueType(0);
3492  SDValue InOp = GetWidenedVector(N->getOperand(0));
3493  EVT InWidenVT = InOp.getValueType();
3494  SDLoc dl(N);
3495 
3496  // Check if we can convert between two legal vector types and extract.
3497  unsigned InWidenSize = InWidenVT.getSizeInBits();
3498  unsigned Size = VT.getSizeInBits();
3499  // x86mmx is not an acceptable vector element type, so don't try.
3500  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
3501  unsigned NewNumElts = InWidenSize / Size;
3502  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
3503  if (TLI.isTypeLegal(NewVT)) {
3504  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
3505  return DAG.getNode(
3506  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
3507  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3508  }
3509  }
3510 
3511  return CreateStackStoreLoad(InOp, VT);
3512 }
3513 
3514 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
3515  // If the input vector is not legal, it is likely that we will not find a
3516  // legal vector of the same size. Replace the concatenate vector with a
3517  // nasty build vector.
3518  EVT VT = N->getValueType(0);
3519  EVT EltVT = VT.getVectorElementType();
3520  SDLoc dl(N);
3521  unsigned NumElts = VT.getVectorNumElements();
3522  SmallVector<SDValue, 16> Ops(NumElts);
3523 
3524  EVT InVT = N->getOperand(0).getValueType();
3525  unsigned NumInElts = InVT.getVectorNumElements();
3526 
3527  unsigned Idx = 0;
3528  unsigned NumOperands = N->getNumOperands();
3529  for (unsigned i=0; i < NumOperands; ++i) {
3530  SDValue InOp = N->getOperand(i);
3531  assert(getTypeAction(InOp.getValueType()) ==
3533  "Unexpected type action");
3534  InOp = GetWidenedVector(InOp);
3535  for (unsigned j=0; j < NumInElts; ++j)
3536  Ops[Idx++] = DAG.getNode(
3537  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3538  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3539  }
3540  return DAG.getBuildVector(VT, dl, Ops);
3541 }
3542 
3543 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3544  SDValue InOp = GetWidenedVector(N->getOperand(0));
3545  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
3546  N->getValueType(0), InOp, N->getOperand(1));
3547 }
3548 
3549 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3550  SDValue InOp = GetWidenedVector(N->getOperand(0));
3551  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3552  N->getValueType(0), InOp, N->getOperand(1));
3553 }
3554 
3555 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
3556  // We have to widen the value, but we want only to store the original
3557  // vector type.
3558  StoreSDNode *ST = cast<StoreSDNode>(N);
3559 
3560  if (!ST->getMemoryVT().getScalarType().isByteSized())
3561  return TLI.scalarizeVectorStore(ST, DAG);
3562 
3563  SmallVector<SDValue, 16> StChain;
3564  if (ST->isTruncatingStore())
3565  GenWidenVectorTruncStores(StChain, ST);
3566  else
3567  GenWidenVectorStores(StChain, ST);
3568 
3569  if (StChain.size() == 1)
3570  return StChain[0];
3571  else
3572  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
3573 }
3574 
3575 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
3576  assert(OpNo == 3 && "Can widen only data operand of mstore");
3577  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
3578  SDValue Mask = MST->getMask();
3579  EVT MaskVT = Mask.getValueType();
3580  SDValue StVal = MST->getValue();
3581  // Widen the value
3582  SDValue WideVal = GetWidenedVector(StVal);
3583  SDLoc dl(N);
3584 
3585  // The mask should be widened as well.
3586  EVT WideVT = WideVal.getValueType();
3587  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3588  MaskVT.getVectorElementType(),
3589  WideVT.getVectorNumElements());
3590  Mask = ModifyToType(Mask, WideMaskVT, true);
3591 
3592  assert(Mask.getValueType().getVectorNumElements() ==
3593  WideVal.getValueType().getVectorNumElements() &&
3594  "Mask and data vectors should have the same number of elements");
3595  return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
3596  Mask, MST->getMemoryVT(), MST->getMemOperand(),
3597  false, MST->isCompressingStore());
3598 }
3599 
3600 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
3601  assert(OpNo == 1 && "Can widen only data operand of mscatter");
3602  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
3603  SDValue DataOp = MSC->getValue();
3604  SDValue Mask = MSC->getMask();
3605  EVT MaskVT = Mask.getValueType();
3606  SDValue Scale = MSC->getScale();
3607 
3608  // Widen the value.
3609  SDValue WideVal = GetWidenedVector(DataOp);
3610  EVT WideVT = WideVal.getValueType();
3611  unsigned NumElts = WideVT.getVectorNumElements();
3612  SDLoc dl(N);
3613 
3614  // The mask should be widened as well.
3615  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3616  MaskVT.getVectorElementType(), NumElts);
3617  Mask = ModifyToType(Mask, WideMaskVT, true);
3618 
3619  // Widen index.
3620  SDValue Index = MSC->getIndex();
3621  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3622  Index.getValueType().getScalarType(),
3623  NumElts);
3624  Index = ModifyToType(Index, WideIndexVT);
3625 
3626  SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index,
3627  Scale};
3628  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
3629  MSC->getMemoryVT(), dl, Ops,
3630  MSC->getMemOperand());
3631 }
3632 
3633 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
3634  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
3635  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3636  SDLoc dl(N);
3637  EVT VT = N->getValueType(0);
3638 
3639  // WARNING: In this code we widen the compare instruction with garbage.
3640  // This garbage may contain denormal floats which may be slow. Is this a real
3641  // concern ? Should we zero the unused lanes if this is a float compare ?
3642 
3643  // Get a new SETCC node to compare the newly widened operands.
3644  // Only some of the compared elements are legal.
3645  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3646  InOp0.getValueType());
3647  // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
3648  if (VT.getScalarType() == MVT::i1)
3649  SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
3650  SVT.getVectorNumElements());
3651 
3652  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
3653  SVT, InOp0, InOp1, N->getOperand(2));
3654 
3655  // Extract the needed results from the result vector.
3656  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
3657  SVT.getVectorElementType(),
3658  VT.getVectorNumElements());
3659  SDValue CC = DAG.getNode(
3660  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
3661  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3662 
3663  return PromoteTargetBoolean(CC, VT);
3664 }
3665 
3666 
3667 //===----------------------------------------------------------------------===//
3668 // Vector Widening Utilities
3669 //===----------------------------------------------------------------------===//
3670 
3671 // Utility function to find the type to chop up a widen vector for load/store
3672 // TLI: Target lowering used to determine legal types.
3673 // Width: Width left need to load/store.
3674 // WidenVT: The widen vector type to load to/store from
3675 // Align: If 0, don't allow use of a wider type
3676 // WidenEx: If Align is not 0, the amount additional we can load/store from.
3677 
3678 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
3679  unsigned Width, EVT WidenVT,
3680  unsigned Align = 0, unsigned WidenEx = 0) {
3681  EVT WidenEltVT = WidenVT.getVectorElementType();
3682  unsigned WidenWidth = WidenVT.getSizeInBits();
3683  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
3684  unsigned AlignInBits = Align*8;
3685 
3686  // If we have one element to load/store, return it.
3687  EVT RetVT = WidenEltVT;
3688  if (Width == WidenEltWidth)
3689  return RetVT;
3690 
3691  // See if there is larger legal integer than the element type to load/store.
3692  unsigned VT;
3693  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
3694  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
3695  EVT MemVT((MVT::SimpleValueType) VT);
3696  unsigned MemVTWidth = MemVT.getSizeInBits();
3697  if (MemVT.getSizeInBits() <= WidenEltWidth)
3698  break;
3699  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
3700  if ((Action == TargetLowering::TypeLegal ||
3702  (WidenWidth % MemVTWidth) == 0 &&
3703  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3704  (MemVTWidth <= Width ||
3705  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3706  RetVT = MemVT;
3707  break;
3708  }
3709  }
3710 
3711  // See if there is a larger vector type to load/store that has the same vector
3712  // element type and is evenly divisible with the WidenVT.
3713  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
3714  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
3715  EVT MemVT = (MVT::SimpleValueType) VT;
3716  unsigned MemVTWidth = MemVT.getSizeInBits();
3717  if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
3718  (WidenWidth % MemVTWidth) == 0 &&
3719  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3720  (MemVTWidth <= Width ||
3721  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3722  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
3723  return MemVT;
3724  }
3725  }
3726 
3727  return RetVT;
3728 }
3729 
3730 // Builds a vector type from scalar loads
3731 // VecTy: Resulting Vector type
3732 // LDOps: Load operators to build a vector type
3733 // [Start,End) the list of loads to use.
3735  SmallVectorImpl<SDValue> &LdOps,
3736  unsigned Start, unsigned End) {
3737  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3738  SDLoc dl(LdOps[Start]);
3739  EVT LdTy = LdOps[Start].getValueType();
3740  unsigned Width = VecTy.getSizeInBits();
3741  unsigned NumElts = Width / LdTy.getSizeInBits();
3742  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
3743 
3744  unsigned Idx = 1;
3745  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
3746 
3747  for (unsigned i = Start + 1; i != End; ++i) {
3748  EVT NewLdTy = LdOps[i].getValueType();
3749  if (NewLdTy != LdTy) {
3750  NumElts = Width / NewLdTy.getSizeInBits();
3751  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
3752  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
3753  // Readjust position and vector position based on new load type.
3754  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
3755  LdTy = NewLdTy;
3756  }
3757  VecOp = DAG.getNode(
3758  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
3759  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3760  }
3761  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
3762 }
3763 
3764 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
3765  LoadSDNode *LD) {
3766  // The strategy assumes that we can efficiently load power-of-two widths.
3767  // The routine chops the vector into the largest vector loads with the same
3768  // element type or scalar loads and then recombines it to the widen vector
3769  // type.
3770  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3771  unsigned WidenWidth = WidenVT.getSizeInBits();
3772  EVT LdVT = LD->getMemoryVT();
3773  SDLoc dl(LD);
3774  assert(LdVT.isVector() && WidenVT.isVector());
3775  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
3776 
3777  // Load information
3778  SDValue Chain = LD->getChain();
3779  SDValue BasePtr = LD->getBasePtr();
3780  unsigned Align = LD->getAlignment();
3781  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3782  AAMDNodes AAInfo = LD->getAAInfo();
3783 
3784  int LdWidth = LdVT.getSizeInBits();
3785  int WidthDiff = WidenWidth - LdWidth;
3786  unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
3787 
3788  // Find the vector type that can load from.
3789  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3790  int NewVTWidth = NewVT.getSizeInBits();
3791  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
3792  Align, MMOFlags, AAInfo);
3793  LdChain.push_back(LdOp.getValue(1));
3794 
3795  // Check if we can load the element with one instruction.
3796  if (LdWidth <= NewVTWidth) {
3797  if (!NewVT.isVector()) {
3798  unsigned NumElts = WidenWidth / NewVTWidth;
3799  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
3800  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
3801  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
3802  }
3803  if (NewVT == WidenVT)
3804  return LdOp;
3805 
3806  assert(WidenWidth % NewVTWidth == 0);
3807  unsigned NumConcat = WidenWidth / NewVTWidth;
3808  SmallVector<SDValue, 16> ConcatOps(NumConcat);
3809  SDValue UndefVal = DAG.getUNDEF(NewVT);
3810  ConcatOps[0] = LdOp;
3811  for (unsigned i = 1; i != NumConcat; ++i)
3812  ConcatOps[i] = UndefVal;
3813  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
3814  }
3815 
3816  // Load vector by using multiple loads from largest vector to scalar.
3818  LdOps.push_back(LdOp);
3819 
3820  LdWidth -= NewVTWidth;
3821  unsigned Offset = 0;
3822 
3823  while (LdWidth > 0) {
3824  unsigned Increment = NewVTWidth / 8;
3825  Offset += Increment;
3826  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
3827 
3828  SDValue L;
3829  if (LdWidth < NewVTWidth) {
3830  // The current type we are using is too large. Find a better size.
3831  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3832  NewVTWidth = NewVT.getSizeInBits();
3833  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3834  LD->getPointerInfo().getWithOffset(Offset),
3835  MinAlign(Align, Increment), MMOFlags, AAInfo);
3836  LdChain.push_back(L.getValue(1));
3837  if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
3838  // Later code assumes the vector loads produced will be mergeable, so we
3839  // must pad the final entry up to the previous width. Scalars are
3840  // combined separately.
3842  Loads.push_back(L);
3843  unsigned size = L->getValueSizeInBits(0);
3844  while (size < LdOp->getValueSizeInBits(0)) {
3845  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
3846  size += L->getValueSizeInBits(0);
3847  }
3848  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
3849  }
3850  } else {
3851  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3852  LD->getPointerInfo().getWithOffset(Offset),
3853  MinAlign(Align, Increment), MMOFlags, AAInfo);
3854  LdChain.push_back(L.getValue(1));
3855  }
3856 
3857  LdOps.push_back(L);
3858  LdOp = L;
3859 
3860  LdWidth -= NewVTWidth;
3861  }
3862 
3863  // Build the vector from the load operations.
3864  unsigned End = LdOps.size();
3865  if (!LdOps[0].getValueType().isVector())
3866  // All the loads are scalar loads.
3867  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
3868 
3869  // If the load contains vectors, build the vector using concat vector.
3870  // All of the vectors used to load are power-of-2, and the scalar loads can be
3871  // combined to make a power-of-2 vector.
3872  SmallVector<SDValue, 16> ConcatOps(End);
3873  int i = End - 1;
3874  int Idx = End;
3875  EVT LdTy = LdOps[i].getValueType();
3876  // First, combine the scalar loads to a vector.
3877  if (!LdTy.isVector()) {
3878  for (--i; i >= 0; --i) {
3879  LdTy = LdOps[i].getValueType();
3880  if (LdTy.isVector())
3881  break;
3882  }
3883  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
3884  }
3885  ConcatOps[--Idx] = LdOps[i];
3886  for (--i; i >= 0; --i) {
3887  EVT NewLdTy = LdOps[i].getValueType();
3888  if (NewLdTy != LdTy) {
3889  // Create a larger vector.
3890  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
3891  makeArrayRef(&ConcatOps[Idx], End - Idx));
3892  Idx = End - 1;
3893  LdTy = NewLdTy;
3894  }
3895  ConcatOps[--Idx] = LdOps[i];
3896  }
3897 
3898  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
3899  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3900  makeArrayRef(&ConcatOps[Idx], End - Idx));
3901 
3902  // We need to fill the rest with undefs to build the vector.
3903  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
3904  SmallVector<SDValue, 16> WidenOps(NumOps);
3905  SDValue UndefVal = DAG.getUNDEF(LdTy);
3906  {
3907  unsigned i = 0;
3908  for (; i != End-Idx; ++i)
3909  WidenOps[i] = ConcatOps[Idx+i];
3910  for (; i != NumOps; ++i)
3911  WidenOps[i] = UndefVal;
3912  }
3913  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
3914 }
3915 
3916 SDValue
3917 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
3918  LoadSDNode *LD,
3919  ISD::LoadExtType ExtType) {
3920  // For extension loads, it may not be more efficient to chop up the vector
3921  // and then extend it. Instead, we unroll the load and build a new vector.
3922  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3923  EVT LdVT = LD->getMemoryVT();
3924  SDLoc dl(LD);
3925  assert(LdVT.isVector() && WidenVT.isVector());
3926 
3927  // Load information
3928  SDValue Chain = LD->getChain();
3929  SDValue BasePtr = LD->getBasePtr();
3930  unsigned Align = LD->getAlignment();
3931  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3932  AAMDNodes AAInfo = LD->getAAInfo();
3933 
3934  EVT EltVT = WidenVT.getVectorElementType();
3935  EVT LdEltVT = LdVT.getVectorElementType();
3936  unsigned NumElts = LdVT.getVectorNumElements();
3937 
3938  // Load each element and widen.
3939  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3940  SmallVector<SDValue, 16> Ops(WidenNumElts);
3941  unsigned Increment = LdEltVT.getSizeInBits() / 8;
3942  Ops[0] =
3943  DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
3944  LdEltVT, Align, MMOFlags, AAInfo);
3945  LdChain.push_back(Ops[0].getValue(1));
3946  unsigned i = 0, Offset = Increment;
3947  for (i=1; i < NumElts; ++i, Offset += Increment) {
3948  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
3949  Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
3950  LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
3951  Align, MMOFlags, AAInfo);
3952  LdChain.push_back(Ops[i].getValue(1));
3953  }
3954 
3955  // Fill the rest with undefs.
3956  SDValue UndefVal = DAG.getUNDEF(EltVT);
3957  for (; i != WidenNumElts; ++i)
3958  Ops[i] = UndefVal;
3959 
3960  return DAG.getBuildVector(WidenVT, dl, Ops);
3961 }
3962 
3963 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
3964  StoreSDNode *ST) {
3965  // The strategy assumes that we can efficiently store power-of-two widths.
3966  // The routine chops the vector into the largest vector stores with the same
3967  // element type or scalar stores.
3968  SDValue Chain = ST->getChain();
3969  SDValue BasePtr = ST->getBasePtr();
3970  unsigned Align = ST->getAlignment();
3971  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
3972  AAMDNodes AAInfo = ST->getAAInfo();
3973  SDValue ValOp = GetWidenedVector(ST->getValue());
3974  SDLoc dl(ST);
3975 
3976  EVT StVT = ST->getMemoryVT();
3977  unsigned StWidth = StVT.getSizeInBits();
3978  EVT ValVT = ValOp.getValueType();
3979  unsigned ValWidth = ValVT.getSizeInBits();
3980  EVT ValEltVT = ValVT.getVectorElementType();
3981  unsigned ValEltWidth = ValEltVT.getSizeInBits();
3982  assert(StVT.getVectorElementType() == ValEltVT);
3983 
3984  int Idx = 0; // current index to store
3985  unsigned Offset = 0; // offset from base to store
3986  while (StWidth != 0) {
3987  // Find the largest vector type we can store with.
3988  EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
3989  unsigned NewVTWidth = NewVT.getSizeInBits();
3990  unsigned Increment = NewVTWidth / 8;
3991  if (NewVT.isVector()) {
3992  unsigned NumVTElts = NewVT.getVectorNumElements();
3993  do {
3994  SDValue EOp = DAG.getNode(
3995  ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
3996  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3997  StChain.push_back(DAG.getStore(
3998  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
3999  MinAlign(Align, Offset), MMOFlags, AAInfo));
4000  StWidth -= NewVTWidth;
4001  Offset += Increment;
4002  Idx += NumVTElts;
4003 
4004  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4005  } while (StWidth != 0 && StWidth >= NewVTWidth);
4006  } else {
4007  // Cast the vector to the scalar type we can store.
4008  unsigned NumElts = ValWidth / NewVTWidth;
4009  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4010  SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4011  // Readjust index position based on new vector type.
4012  Idx = Idx * ValEltWidth / NewVTWidth;
4013  do {
4014  SDValue EOp = DAG.getNode(
4015  ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4016  DAG.getConstant(Idx++, dl,
4017  TLI.getVectorIdxTy(DAG.getDataLayout())));
4018  StChain.push_back(DAG.getStore(
4019  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4020  MinAlign(Align, Offset), MMOFlags, AAInfo));
4021  StWidth -= NewVTWidth;
4022  Offset += Increment;
4023  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4024  } while (StWidth != 0 && StWidth >= NewVTWidth);
4025  // Restore index back to be relative to the original widen element type.
4026  Idx = Idx * NewVTWidth / ValEltWidth;
4027  }
4028  }
4029 }
4030 
4031 void
4032 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4033  StoreSDNode *ST) {
4034  // For extension loads, it may not be more efficient to truncate the vector
4035  // and then store it. Instead, we extract each element and then store it.
4036  SDValue Chain = ST->getChain();
4037  SDValue BasePtr = ST->getBasePtr();
4038  unsigned Align = ST->getAlignment();
4039  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4040  AAMDNodes AAInfo = ST->getAAInfo();
4041  SDValue ValOp = GetWidenedVector(ST->getValue());
4042  SDLoc dl(ST);
4043 
4044  EVT StVT = ST->getMemoryVT();
4045  EVT ValVT = ValOp.getValueType();
4046 
4047  // It must be true that the wide vector type is bigger than where we need to
4048  // store.
4049  assert(StVT.isVector() && ValOp.getValueType().isVector());
4050  assert(StVT.bitsLT(ValOp.getValueType()));
4051 
4052  // For truncating stores, we can not play the tricks of chopping legal vector
4053  // types and bitcast it to the right type. Instead, we unroll the store.
4054  EVT StEltVT = StVT.getVectorElementType();
4055  EVT ValEltVT = ValVT.getVectorElementType();
4056  unsigned Increment = ValEltVT.getSizeInBits() / 8;
4057  unsigned NumElts = StVT.getVectorNumElements();
4058  SDValue EOp = DAG.getNode(
4059  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4060  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4061  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4062  ST->getPointerInfo(), StEltVT, Align,
4063  MMOFlags, AAInfo));
4064  unsigned Offset = Increment;
4065  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4066  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4067  SDValue EOp = DAG.getNode(
4068  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4069  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4070  StChain.push_back(DAG.getTruncStore(
4071  Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4072  StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4073  }
4074 }
4075 
4076 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
4077 /// input vector must have the same element type as NVT.
4078 /// FillWithZeroes specifies that the vector should be widened with zeroes.
4079 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
4080  bool FillWithZeroes) {
4081  // Note that InOp might have been widened so it might already have
4082  // the right width or it might need be narrowed.
4083  EVT InVT = InOp.getValueType();
4085  "input and widen element type must match");
4086  SDLoc dl(InOp);
4087 
4088  // Check if InOp already has the right width.
4089  if (InVT == NVT)
4090  return InOp;
4091 
4092  unsigned InNumElts = InVT.getVectorNumElements();
4093  unsigned WidenNumElts = NVT.getVectorNumElements();
4094  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
4095  unsigned NumConcat = WidenNumElts / InNumElts;
4096  SmallVector<SDValue, 16> Ops(NumConcat);
4097  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
4098  DAG.getUNDEF(InVT);
4099  Ops[0] = InOp;
4100  for (unsigned i = 1; i != NumConcat; ++i)
4101  Ops[i] = FillVal;
4102 
4103  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
4104  }
4105 
4106  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
4107  return DAG.getNode(
4108  ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
4109  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4110 
4111  // Fall back to extract and build.
4112  SmallVector<SDValue, 16> Ops(WidenNumElts);
4113  EVT EltVT = NVT.getVectorElementType();
4114  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
4115  unsigned Idx;
4116  for (Idx = 0; Idx < MinNumElts; ++Idx)
4117  Ops[Idx] = DAG.getNode(
4118  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4119  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4120 
4121  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
4122  DAG.getUNDEF(EltVT);
4123  for ( ; Idx < WidenNumElts; ++Idx)
4124  Ops[Idx] = FillVal;
4125  return DAG.getBuildVector(NVT, dl, Ops);
4126 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:546
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:513
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:570
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isUndef() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:334
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:115
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:343
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:495
SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:351
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getBasePtr() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getChain() const
unsigned getAlignment() const
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:932
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:486
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:360
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:329
uint64_t High
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:405
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
const SDValue & getValue() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
A description of a memory reference used in the backend.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:380
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:205
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:449
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:390
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:387
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:456
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
const SDValue & getScale() const
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:399
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:851
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:561
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:473
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:499
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:303
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:229
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:602
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getBasePtr() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
These reductions are non-strict, and have a single vector operand.
Definition: ISDOpcodes.h:826
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:421
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:69
Simple binary floating point operators.
Definition: ISDOpcodes.h:260
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:892
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:308
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node&#39;s operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:692
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:426
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Idx) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition: ValueTypes.h:58
bool hasNoNaNs() const
static const unsigned End
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:386
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:531
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
const SDValue & getSrc0() const
bool isVolatile() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const SDValue & getMask() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
The memory access writes data.
SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:393
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:295
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:315
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:210
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:528
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:862
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:715
bool isVector(MCInstrInfo const &MCII, MCInst const &MCI)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:721
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:96
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:642
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:383
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
Definition: ValueTypes.h:342
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored...
Represents one node in the SelectionDAG.
static bool isSETCCorConvertedSETCC(SDValue N)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachinePointerInfo getWithOffset(int64_t O) const
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:923
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:390
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:446
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:449
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:288
int getMaskElt(unsigned Idx) const
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
Definition: ISDOpcodes.h:573
Flags
Flags values. These may be or&#39;d together.
amdgpu Simplify well known AMD library false Value Value * Arg
The memory access reads data.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
pointer data()
Return a pointer to the vector&#39;s buffer, even if empty().
Definition: SmallVector.h:143
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:464
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:582
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return &#39;Legal&#39;) or we ...
#define N
Flags getFlags() const
Return the raw flags of the source value,.
unsigned getOpcode() const
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:321
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:278
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:831
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
bool isUndef() const
Return true if the type of the node type undefined.
#define DEBUG(X)
Definition: Debug.h:118
ISD::LoadExtType getExtensionType() const
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:413
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
static bool isLogicalMaskOp(unsigned Opcode)
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:198
Conversion operators.
Definition: ISDOpcodes.h:443
BooleanContent
Enum that describes how the target represents true/false values.
const SDValue & getOperand(unsigned i) const
bool isBigEndian() const
Definition: DataLayout.h:217
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:452
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Definition: SelectionDAG.h:805
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:562
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVMContext * getContext() const
Definition: SelectionDAG.h:396
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:351
const SDValue & getMask() const
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:356
This class is used to represent ISD::LOAD nodes.