LLVM  6.0.0svn
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16 // types.
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "legalize-types"
30 
31 //===----------------------------------------------------------------------===//
32 // Result Vector Scalarization: <1 x ty> -> ty.
33 //===----------------------------------------------------------------------===//
34 
35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36  DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
37  N->dump(&DAG);
38  dbgs() << "\n");
39  SDValue R = SDValue();
40 
41  switch (N->getOpcode()) {
42  default:
43 #ifndef NDEBUG
44  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45  N->dump(&DAG);
46  dbgs() << "\n";
47 #endif
48  report_fatal_error("Do not know how to scalarize the result of this "
49  "operator!\n");
50 
51  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
57  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
58  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
59  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
60  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
61  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
62  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
63  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
64  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
65  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
66  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
67  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71  R = ScalarizeVecRes_VecInregOp(N);
72  break;
73  case ISD::ANY_EXTEND:
74  case ISD::BITREVERSE:
75  case ISD::BSWAP:
76  case ISD::CTLZ:
78  case ISD::CTPOP:
79  case ISD::CTTZ:
81  case ISD::FABS:
82  case ISD::FCEIL:
83  case ISD::FCOS:
84  case ISD::FEXP:
85  case ISD::FEXP2:
86  case ISD::FFLOOR:
87  case ISD::FLOG:
88  case ISD::FLOG10:
89  case ISD::FLOG2:
90  case ISD::FNEARBYINT:
91  case ISD::FNEG:
92  case ISD::FP_EXTEND:
93  case ISD::FP_TO_SINT:
94  case ISD::FP_TO_UINT:
95  case ISD::FRINT:
96  case ISD::FROUND:
97  case ISD::FSIN:
98  case ISD::FSQRT:
99  case ISD::FTRUNC:
100  case ISD::SIGN_EXTEND:
101  case ISD::SINT_TO_FP:
102  case ISD::TRUNCATE:
103  case ISD::UINT_TO_FP:
104  case ISD::ZERO_EXTEND:
105  case ISD::FCANONICALIZE:
106  R = ScalarizeVecRes_UnaryOp(N);
107  break;
108 
109  case ISD::ADD:
110  case ISD::AND:
111  case ISD::FADD:
112  case ISD::FCOPYSIGN:
113  case ISD::FDIV:
114  case ISD::FMUL:
115  case ISD::FMINNUM:
116  case ISD::FMAXNUM:
117  case ISD::FMINNAN:
118  case ISD::FMAXNAN:
119  case ISD::SMIN:
120  case ISD::SMAX:
121  case ISD::UMIN:
122  case ISD::UMAX:
123 
124  case ISD::FPOW:
125  case ISD::FREM:
126  case ISD::FSUB:
127  case ISD::MUL:
128  case ISD::OR:
129  case ISD::SDIV:
130  case ISD::SREM:
131  case ISD::SUB:
132  case ISD::UDIV:
133  case ISD::UREM:
134  case ISD::XOR:
135  case ISD::SHL:
136  case ISD::SRA:
137  case ISD::SRL:
138  R = ScalarizeVecRes_BinOp(N);
139  break;
140  case ISD::FMA:
141  R = ScalarizeVecRes_TernaryOp(N);
142  break;
143  }
144 
145  // If R is null, the sub-method took care of registering the result.
146  if (R.getNode())
147  SetScalarizedVector(SDValue(N, ResNo), R);
148 }
149 
150 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
151  SDValue LHS = GetScalarizedVector(N->getOperand(0));
152  SDValue RHS = GetScalarizedVector(N->getOperand(1));
153  return DAG.getNode(N->getOpcode(), SDLoc(N),
154  LHS.getValueType(), LHS, RHS, N->getFlags());
155 }
156 
157 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
158  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
159  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
160  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
161  return DAG.getNode(N->getOpcode(), SDLoc(N),
162  Op0.getValueType(), Op0, Op1, Op2);
163 }
164 
165 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
166  unsigned ResNo) {
167  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
168  return GetScalarizedVector(Op);
169 }
170 
171 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
172  EVT NewVT = N->getValueType(0).getVectorElementType();
173  return DAG.getNode(ISD::BITCAST, SDLoc(N),
174  NewVT, N->getOperand(0));
175 }
176 
177 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
178  EVT EltVT = N->getValueType(0).getVectorElementType();
179  SDValue InOp = N->getOperand(0);
180  // The BUILD_VECTOR operands may be of wider element types and
181  // we may need to truncate them back to the requested return type.
182  if (EltVT.isInteger())
183  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
184  return InOp;
185 }
186 
187 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
188  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
190  N->getOperand(0), N->getOperand(1));
191 }
192 
193 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
194  EVT NewVT = N->getValueType(0).getVectorElementType();
195  SDValue Op = GetScalarizedVector(N->getOperand(0));
196  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
197  NewVT, Op, N->getOperand(1));
198 }
199 
200 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
201  SDValue Op = GetScalarizedVector(N->getOperand(0));
202  return DAG.getNode(ISD::FPOWI, SDLoc(N),
203  Op.getValueType(), Op, N->getOperand(1));
204 }
205 
206 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
207  // The value to insert may have a wider type than the vector element type,
208  // so be sure to truncate it to the element type if necessary.
209  SDValue Op = N->getOperand(1);
210  EVT EltVT = N->getValueType(0).getVectorElementType();
211  if (Op.getValueType() != EltVT)
212  // FIXME: Can this happen for floating point types?
213  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
214  return Op;
215 }
216 
217 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
218  assert(N->isUnindexed() && "Indexed vector load?");
219 
220  SDValue Result = DAG.getLoad(
223  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
226  N->getAAInfo());
227 
228  // Legalize the chain result - switch anything that used the old chain to
229  // use the new one.
230  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
231  return Result;
232 }
233 
234 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
235  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
236  EVT DestVT = N->getValueType(0).getVectorElementType();
237  SDValue Op = N->getOperand(0);
238  EVT OpVT = Op.getValueType();
239  SDLoc DL(N);
240  // The result needs scalarizing, but it's not a given that the source does.
241  // This is a workaround for targets where it's impossible to scalarize the
242  // result of a conversion, because the source type is legal.
243  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
244  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
245  // legal and was not scalarized.
246  // See the similar logic in ScalarizeVecRes_VSETCC
247  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
248  Op = GetScalarizedVector(Op);
249  } else {
250  EVT VT = OpVT.getVectorElementType();
251  Op = DAG.getNode(
252  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
253  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
254  }
255  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
256 }
257 
258 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
259  EVT EltVT = N->getValueType(0).getVectorElementType();
260  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
261  SDValue LHS = GetScalarizedVector(N->getOperand(0));
262  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
263  LHS, DAG.getValueType(ExtVT));
264 }
265 
266 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
267  SDLoc DL(N);
268  SDValue Op = N->getOperand(0);
269 
270  EVT OpVT = Op.getValueType();
271  EVT OpEltVT = OpVT.getVectorElementType();
272  EVT EltVT = N->getValueType(0).getVectorElementType();
273 
274  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
275  Op = GetScalarizedVector(Op);
276  } else {
277  Op = DAG.getNode(
278  ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
279  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
280  }
281 
282  switch (N->getOpcode()) {
284  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
286  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
288  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
289  }
290 
291  llvm_unreachable("Illegal extend_vector_inreg opcode");
292 }
293 
294 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
295  // If the operand is wider than the vector element type then it is implicitly
296  // truncated. Make that explicit here.
297  EVT EltVT = N->getValueType(0).getVectorElementType();
298  SDValue InOp = N->getOperand(0);
299  if (InOp.getValueType() != EltVT)
300  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
301  return InOp;
302 }
303 
304 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
305  SDValue Cond = N->getOperand(0);
306  EVT OpVT = Cond.getValueType();
307  SDLoc DL(N);
308  // The vselect result and true/value operands needs scalarizing, but it's
309  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
310  // See the similar logic in ScalarizeVecRes_VSETCC
311  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
312  Cond = GetScalarizedVector(Cond);
313  } else {
314  EVT VT = OpVT.getVectorElementType();
315  Cond = DAG.getNode(
316  ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
317  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
318  }
319 
320  SDValue LHS = GetScalarizedVector(N->getOperand(1));
321  TargetLowering::BooleanContent ScalarBool =
322  TLI.getBooleanContents(false, false);
323  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
324 
325  // If integer and float booleans have different contents then we can't
326  // reliably optimize in all cases. There is a full explanation for this in
327  // DAGCombiner::visitSELECT() where the same issue affects folding
328  // (select C, 0, 1) to (xor C, 1).
329  if (TLI.getBooleanContents(false, false) !=
330  TLI.getBooleanContents(false, true)) {
331  // At least try the common case where the boolean is generated by a
332  // comparison.
333  if (Cond->getOpcode() == ISD::SETCC) {
334  EVT OpVT = Cond->getOperand(0)->getValueType(0);
335  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
336  VecBool = TLI.getBooleanContents(OpVT);
337  } else
339  }
340 
341  if (ScalarBool != VecBool) {
342  EVT CondVT = Cond.getValueType();
343  switch (ScalarBool) {
345  break;
349  // Vector read from all ones, scalar expects a single 1 so mask.
350  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
351  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
352  break;
356  // Vector reads from a one, scalar from all ones so sign extend.
357  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
358  Cond, DAG.getValueType(MVT::i1));
359  break;
360  }
361  }
362 
363  return DAG.getSelect(SDLoc(N),
364  LHS.getValueType(), Cond, LHS,
365  GetScalarizedVector(N->getOperand(2)));
366 }
367 
368 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
369  SDValue LHS = GetScalarizedVector(N->getOperand(1));
370  return DAG.getSelect(SDLoc(N),
371  LHS.getValueType(), N->getOperand(0), LHS,
372  GetScalarizedVector(N->getOperand(2)));
373 }
374 
375 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
376  SDValue LHS = GetScalarizedVector(N->getOperand(2));
377  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
378  N->getOperand(0), N->getOperand(1),
379  LHS, GetScalarizedVector(N->getOperand(3)),
380  N->getOperand(4));
381 }
382 
383 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
384  assert(N->getValueType(0).isVector() ==
385  N->getOperand(0).getValueType().isVector() &&
386  "Scalar/Vector type mismatch");
387 
388  if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N);
389 
390  SDValue LHS = GetScalarizedVector(N->getOperand(0));
391  SDValue RHS = GetScalarizedVector(N->getOperand(1));
392  SDLoc DL(N);
393 
394  // Turn it into a scalar SETCC.
395  return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2));
396 }
397 
398 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
399  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
400 }
401 
402 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
403  // Figure out if the scalar is the LHS or RHS and return it.
404  SDValue Arg = N->getOperand(2).getOperand(0);
405  if (Arg.isUndef())
406  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
407  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
408  return GetScalarizedVector(N->getOperand(Op));
409 }
410 
411 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
412  assert(N->getValueType(0).isVector() &&
413  N->getOperand(0).getValueType().isVector() &&
414  "Operand types must be vectors");
415  SDValue LHS = N->getOperand(0);
416  SDValue RHS = N->getOperand(1);
417  EVT OpVT = LHS.getValueType();
418  EVT NVT = N->getValueType(0).getVectorElementType();
419  SDLoc DL(N);
420 
421  // The result needs scalarizing, but it's not a given that the source does.
422  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
423  LHS = GetScalarizedVector(LHS);
424  RHS = GetScalarizedVector(RHS);
425  } else {
426  EVT VT = OpVT.getVectorElementType();
427  LHS = DAG.getNode(
428  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
429  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
430  RHS = DAG.getNode(
431  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
432  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
433  }
434 
435  // Turn it into a scalar SETCC.
436  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
437  N->getOperand(2));
438  // Vectors may have a different boolean contents to scalars. Promote the
439  // value appropriately.
440  ISD::NodeType ExtendCode =
442  return DAG.getNode(ExtendCode, DL, NVT, Res);
443 }
444 
445 
446 //===----------------------------------------------------------------------===//
447 // Operand Vector Scalarization <1 x ty> -> ty.
448 //===----------------------------------------------------------------------===//
449 
450 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
451  DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
452  N->dump(&DAG);
453  dbgs() << "\n");
454  SDValue Res = SDValue();
455 
456  if (!Res.getNode()) {
457  switch (N->getOpcode()) {
458  default:
459 #ifndef NDEBUG
460  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
461  N->dump(&DAG);
462  dbgs() << "\n";
463 #endif
464  report_fatal_error("Do not know how to scalarize this operator's "
465  "operand!\n");
466  case ISD::BITCAST:
467  Res = ScalarizeVecOp_BITCAST(N);
468  break;
469  case ISD::ANY_EXTEND:
470  case ISD::ZERO_EXTEND:
471  case ISD::SIGN_EXTEND:
472  case ISD::TRUNCATE:
473  case ISD::FP_TO_SINT:
474  case ISD::FP_TO_UINT:
475  case ISD::SINT_TO_FP:
476  case ISD::UINT_TO_FP:
477  Res = ScalarizeVecOp_UnaryOp(N);
478  break;
479  case ISD::CONCAT_VECTORS:
480  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
481  break;
483  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
484  break;
485  case ISD::VSELECT:
486  Res = ScalarizeVecOp_VSELECT(N);
487  break;
488  case ISD::SETCC:
489  Res = ScalarizeVecOp_VSETCC(N);
490  break;
491  case ISD::STORE:
492  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
493  break;
494  case ISD::FP_ROUND:
495  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
496  break;
497  }
498  }
499 
500  // If the result is null, the sub-method took care of registering results etc.
501  if (!Res.getNode()) return false;
502 
503  // If the result is N, the sub-method updated N in place. Tell the legalizer
504  // core about this.
505  if (Res.getNode() == N)
506  return true;
507 
508  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
509  "Invalid operand expansion");
510 
511  ReplaceValueWith(SDValue(N, 0), Res);
512  return false;
513 }
514 
515 /// If the value to convert is a vector that needs to be scalarized, it must be
516 /// <1 x ty>. Convert the element instead.
517 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
518  SDValue Elt = GetScalarizedVector(N->getOperand(0));
519  return DAG.getNode(ISD::BITCAST, SDLoc(N),
520  N->getValueType(0), Elt);
521 }
522 
523 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
524 /// Do the operation on the element instead.
525 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
527  "Unexpected vector type!");
528  SDValue Elt = GetScalarizedVector(N->getOperand(0));
529  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
530  N->getValueType(0).getScalarType(), Elt);
531  // Revectorize the result so the types line up with what the uses of this
532  // expression expect.
533  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Op);
534 }
535 
536 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
537 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
539  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
540  Ops[i] = GetScalarizedVector(N->getOperand(i));
541  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
542 }
543 
544 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
545 /// so just return the element, ignoring the index.
546 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
547  EVT VT = N->getValueType(0);
548  SDValue Res = GetScalarizedVector(N->getOperand(0));
549  if (Res.getValueType() != VT)
550  Res = VT.isFloatingPoint()
551  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
552  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
553  return Res;
554 }
555 
556 /// If the input condition is a vector that needs to be scalarized, it must be
557 /// <1 x i1>, so just convert to a normal ISD::SELECT
558 /// (still with vector output type since that was acceptable if we got here).
559 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
560  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
561  EVT VT = N->getValueType(0);
562 
563  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
564  N->getOperand(2));
565 }
566 
567 /// If the operand is a vector that needs to be scalarized then the
568 /// result must be v1i1, so just convert to a scalar SETCC and wrap
569 /// with a scalar_to_vector since the res type is legal if we got here
570 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
571  assert(N->getValueType(0).isVector() &&
572  N->getOperand(0).getValueType().isVector() &&
573  "Operand types must be vectors");
574  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
575 
576  EVT VT = N->getValueType(0);
577  SDValue LHS = GetScalarizedVector(N->getOperand(0));
578  SDValue RHS = GetScalarizedVector(N->getOperand(1));
579 
580  EVT OpVT = N->getOperand(0).getValueType();
581  EVT NVT = VT.getVectorElementType();
582  SDLoc DL(N);
583  // Turn it into a scalar SETCC.
584  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
585  N->getOperand(2));
586 
587  // Vectors may have a different boolean contents to scalars. Promote the
588  // value appropriately.
589  ISD::NodeType ExtendCode =
591 
592  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
593 
594  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
595 }
596 
597 /// If the value to store is a vector that needs to be scalarized, it must be
598 /// <1 x ty>. Just store the element.
599 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
600  assert(N->isUnindexed() && "Indexed store of one-element vector?");
601  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
602  SDLoc dl(N);
603 
604  if (N->isTruncatingStore())
605  return DAG.getTruncStore(
606  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
607  N->getBasePtr(), N->getPointerInfo(),
609  N->getMemOperand()->getFlags(), N->getAAInfo());
610 
611  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
612  N->getBasePtr(), N->getPointerInfo(),
614  N->getAAInfo());
615 }
616 
617 /// If the value to round is a vector that needs to be scalarized, it must be
618 /// <1 x ty>. Convert the element instead.
619 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
620  SDValue Elt = GetScalarizedVector(N->getOperand(0));
621  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
622  N->getValueType(0).getVectorElementType(), Elt,
623  N->getOperand(1));
624  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
625 }
626 
627 //===----------------------------------------------------------------------===//
628 // Result Vector Splitting
629 //===----------------------------------------------------------------------===//
630 
631 /// This method is called when the specified result of the specified node is
632 /// found to need vector splitting. At this point, the node may also have
633 /// invalid operands or may have other results that need legalization, we just
634 /// know that (at least) one result needs vector splitting.
635 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
636  DEBUG(dbgs() << "Split node result: ";
637  N->dump(&DAG);
638  dbgs() << "\n");
639  SDValue Lo, Hi;
640 
641  // See if the target wants to custom expand this node.
642  if (CustomLowerNode(N, N->getValueType(ResNo), true))
643  return;
644 
645  switch (N->getOpcode()) {
646  default:
647 #ifndef NDEBUG
648  dbgs() << "SplitVectorResult #" << ResNo << ": ";
649  N->dump(&DAG);
650  dbgs() << "\n";
651 #endif
652  report_fatal_error("Do not know how to split the result of this "
653  "operator!\n");
654 
655  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
656  case ISD::VSELECT:
657  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
658  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
659  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
660  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
661  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
662  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
663  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
664  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
665  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
666  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
667  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
668  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
669  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
670  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
671  case ISD::LOAD:
672  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
673  break;
674  case ISD::MLOAD:
675  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
676  break;
677  case ISD::MGATHER:
678  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
679  break;
680  case ISD::SETCC:
681  SplitVecRes_SETCC(N, Lo, Hi);
682  break;
683  case ISD::VECTOR_SHUFFLE:
684  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
685  break;
686 
690  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
691  break;
692 
693  case ISD::BITREVERSE:
694  case ISD::BSWAP:
695  case ISD::CTLZ:
696  case ISD::CTTZ:
699  case ISD::CTPOP:
700  case ISD::FABS:
701  case ISD::FCEIL:
702  case ISD::FCOS:
703  case ISD::FEXP:
704  case ISD::FEXP2:
705  case ISD::FFLOOR:
706  case ISD::FLOG:
707  case ISD::FLOG10:
708  case ISD::FLOG2:
709  case ISD::FNEARBYINT:
710  case ISD::FNEG:
711  case ISD::FP_EXTEND:
712  case ISD::FP_ROUND:
713  case ISD::FP_TO_SINT:
714  case ISD::FP_TO_UINT:
715  case ISD::FRINT:
716  case ISD::FROUND:
717  case ISD::FSIN:
718  case ISD::FSQRT:
719  case ISD::FTRUNC:
720  case ISD::SINT_TO_FP:
721  case ISD::TRUNCATE:
722  case ISD::UINT_TO_FP:
723  case ISD::FCANONICALIZE:
724  SplitVecRes_UnaryOp(N, Lo, Hi);
725  break;
726 
727  case ISD::ANY_EXTEND:
728  case ISD::SIGN_EXTEND:
729  case ISD::ZERO_EXTEND:
730  SplitVecRes_ExtendOp(N, Lo, Hi);
731  break;
732 
733  case ISD::ADD:
734  case ISD::SUB:
735  case ISD::MUL:
736  case ISD::MULHS:
737  case ISD::MULHU:
738  case ISD::FADD:
739  case ISD::FSUB:
740  case ISD::FMUL:
741  case ISD::FMINNUM:
742  case ISD::FMAXNUM:
743  case ISD::FMINNAN:
744  case ISD::FMAXNAN:
745  case ISD::SDIV:
746  case ISD::UDIV:
747  case ISD::FDIV:
748  case ISD::FPOW:
749  case ISD::AND:
750  case ISD::OR:
751  case ISD::XOR:
752  case ISD::SHL:
753  case ISD::SRA:
754  case ISD::SRL:
755  case ISD::UREM:
756  case ISD::SREM:
757  case ISD::FREM:
758  case ISD::SMIN:
759  case ISD::SMAX:
760  case ISD::UMIN:
761  case ISD::UMAX:
762  SplitVecRes_BinOp(N, Lo, Hi);
763  break;
764  case ISD::FMA:
765  SplitVecRes_TernaryOp(N, Lo, Hi);
766  break;
767  }
768 
769  // If Lo/Hi is null, the sub-method took care of registering results etc.
770  if (Lo.getNode())
771  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
772 }
773 
774 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
775  SDValue &Hi) {
776  SDValue LHSLo, LHSHi;
777  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
778  SDValue RHSLo, RHSHi;
779  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
780  SDLoc dl(N);
781 
782  const SDNodeFlags Flags = N->getFlags();
783  unsigned Opcode = N->getOpcode();
784  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
785  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
786 }
787 
788 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
789  SDValue &Hi) {
790  SDValue Op0Lo, Op0Hi;
791  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
792  SDValue Op1Lo, Op1Hi;
793  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
794  SDValue Op2Lo, Op2Hi;
795  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
796  SDLoc dl(N);
797 
798  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
799  Op0Lo, Op1Lo, Op2Lo);
800  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
801  Op0Hi, Op1Hi, Op2Hi);
802 }
803 
804 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
805  SDValue &Hi) {
806  // We know the result is a vector. The input may be either a vector or a
807  // scalar value.
808  EVT LoVT, HiVT;
809  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
810  SDLoc dl(N);
811 
812  SDValue InOp = N->getOperand(0);
813  EVT InVT = InOp.getValueType();
814 
815  // Handle some special cases efficiently.
816  switch (getTypeAction(InVT)) {
823  break;
826  // A scalar to vector conversion, where the scalar needs expansion.
827  // If the vector is being split in two then we can just convert the
828  // expanded pieces.
829  if (LoVT == HiVT) {
830  GetExpandedOp(InOp, Lo, Hi);
831  if (DAG.getDataLayout().isBigEndian())
832  std::swap(Lo, Hi);
833  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
834  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
835  return;
836  }
837  break;
839  // If the input is a vector that needs to be split, convert each split
840  // piece of the input now.
841  GetSplitVector(InOp, Lo, Hi);
842  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
843  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
844  return;
845  }
846 
847  // In the general case, convert the input to an integer and split it by hand.
848  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
849  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
850  if (DAG.getDataLayout().isBigEndian())
851  std::swap(LoIntVT, HiIntVT);
852 
853  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
854 
855  if (DAG.getDataLayout().isBigEndian())
856  std::swap(Lo, Hi);
857  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
858  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
859 }
860 
861 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
862  SDValue &Hi) {
863  EVT LoVT, HiVT;
864  SDLoc dl(N);
865  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
866  unsigned LoNumElts = LoVT.getVectorNumElements();
867  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
868  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
869 
870  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
871  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
872 }
873 
874 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
875  SDValue &Hi) {
876  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
877  SDLoc dl(N);
878  unsigned NumSubvectors = N->getNumOperands() / 2;
879  if (NumSubvectors == 1) {
880  Lo = N->getOperand(0);
881  Hi = N->getOperand(1);
882  return;
883  }
884 
885  EVT LoVT, HiVT;
886  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
887 
888  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
889  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
890 
891  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
892  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
893 }
894 
895 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
896  SDValue &Hi) {
897  SDValue Vec = N->getOperand(0);
898  SDValue Idx = N->getOperand(1);
899  SDLoc dl(N);
900 
901  EVT LoVT, HiVT;
902  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
903 
904  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
905  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
906  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
907  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
908  TLI.getVectorIdxTy(DAG.getDataLayout())));
909 }
910 
911 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
912  SDValue &Hi) {
913  SDValue Vec = N->getOperand(0);
914  SDValue SubVec = N->getOperand(1);
915  SDValue Idx = N->getOperand(2);
916  SDLoc dl(N);
917  GetSplitVector(Vec, Lo, Hi);
918 
919  EVT VecVT = Vec.getValueType();
920  unsigned VecElems = VecVT.getVectorNumElements();
921  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
922 
923  // If we know the index is 0, and we know the subvector doesn't cross the
924  // boundary between the halves, we can avoid spilling the vector, and insert
925  // into the lower half of the split vector directly.
926  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
927  // the index is constant and there is no boundary crossing. But those cases
928  // don't seem to get hit in practice.
929  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
930  unsigned IdxVal = ConstIdx->getZExtValue();
931  if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
932  EVT LoVT, HiVT;
933  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
934  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
935  return;
936  }
937  }
938 
939  // Spill the vector to the stack.
940  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
941  SDValue Store =
942  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
943 
944  // Store the new subvector into the specified index.
945  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
946  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
947  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
948  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
949 
950  // Load the Lo part from the stack slot.
951  Lo =
952  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
953 
954  // Increment the pointer to the other part.
955  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
956  StackPtr =
957  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
958  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
959 
960  // Load the Hi part from the stack slot.
961  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
962  MinAlign(Alignment, IncrementSize));
963 }
964 
965 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
966  SDValue &Hi) {
967  SDLoc dl(N);
968  GetSplitVector(N->getOperand(0), Lo, Hi);
969  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
970  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
971 }
972 
973 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
974  SDValue &Hi) {
975  SDValue LHSLo, LHSHi;
976  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
977  SDLoc DL(N);
978 
979  SDValue RHSLo, RHSHi;
980  SDValue RHS = N->getOperand(1);
981  EVT RHSVT = RHS.getValueType();
982  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
983  GetSplitVector(RHS, RHSLo, RHSHi);
984  else
985  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
986 
987 
988  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
989  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
990 }
991 
992 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
993  SDValue &Hi) {
994  SDValue LHSLo, LHSHi;
995  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
996  SDLoc dl(N);
997 
998  EVT LoVT, HiVT;
999  std::tie(LoVT, HiVT) =
1000  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1001 
1002  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1003  DAG.getValueType(LoVT));
1004  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1005  DAG.getValueType(HiVT));
1006 }
1007 
1008 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1009  SDValue &Hi) {
1010  unsigned Opcode = N->getOpcode();
1011  SDValue N0 = N->getOperand(0);
1012 
1013  SDLoc dl(N);
1014  SDValue InLo, InHi;
1015 
1016  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1017  GetSplitVector(N0, InLo, InHi);
1018  else
1019  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1020 
1021  EVT InLoVT = InLo.getValueType();
1022  unsigned InNumElements = InLoVT.getVectorNumElements();
1023 
1024  EVT OutLoVT, OutHiVT;
1025  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1026  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1027  assert((2 * OutNumElements) <= InNumElements &&
1028  "Illegal extend vector in reg split");
1029 
1030  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1031  // input vector (i.e. we only use InLo):
1032  // OutLo will extend the first OutNumElements from InLo.
1033  // OutHi will extend the next OutNumElements from InLo.
1034 
1035  // Shuffle the elements from InLo for OutHi into the bottom elements to
1036  // create a 'fake' InHi.
1037  SmallVector<int, 8> SplitHi(InNumElements, -1);
1038  for (unsigned i = 0; i != OutNumElements; ++i)
1039  SplitHi[i] = i + OutNumElements;
1040  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1041 
1042  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1043  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1044 }
1045 
1046 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1047  SDValue &Hi) {
1048  SDValue Vec = N->getOperand(0);
1049  SDValue Elt = N->getOperand(1);
1050  SDValue Idx = N->getOperand(2);
1051  SDLoc dl(N);
1052  GetSplitVector(Vec, Lo, Hi);
1053 
1054  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1055  unsigned IdxVal = CIdx->getZExtValue();
1056  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1057  if (IdxVal < LoNumElts)
1058  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1059  Lo.getValueType(), Lo, Elt, Idx);
1060  else
1061  Hi =
1062  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1063  DAG.getConstant(IdxVal - LoNumElts, dl,
1064  TLI.getVectorIdxTy(DAG.getDataLayout())));
1065  return;
1066  }
1067 
1068  // See if the target wants to custom expand this node.
1069  if (CustomLowerNode(N, N->getValueType(0), true))
1070  return;
1071 
1072  // Spill the vector to the stack.
1073  EVT VecVT = Vec.getValueType();
1074  EVT EltVT = VecVT.getVectorElementType();
1075  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1076  SDValue Store =
1077  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1078 
1079  // Store the new element. This may be larger than the vector element type,
1080  // so use a truncating store.
1081  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1082  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1083  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1084  Store =
1085  DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT);
1086 
1087  // Load the Lo part from the stack slot.
1088  Lo =
1089  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1090 
1091  // Increment the pointer to the other part.
1092  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1093  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1094  DAG.getConstant(IncrementSize, dl,
1095  StackPtr.getValueType()));
1096 
1097  // Load the Hi part from the stack slot.
1098  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1099  MinAlign(Alignment, IncrementSize));
1100 }
1101 
1102 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1103  SDValue &Hi) {
1104  EVT LoVT, HiVT;
1105  SDLoc dl(N);
1106  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1107  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1108  Hi = DAG.getUNDEF(HiVT);
1109 }
1110 
1111 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1112  SDValue &Hi) {
1113  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1114  EVT LoVT, HiVT;
1115  SDLoc dl(LD);
1116  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1117 
1118  ISD::LoadExtType ExtType = LD->getExtensionType();
1119  SDValue Ch = LD->getChain();
1120  SDValue Ptr = LD->getBasePtr();
1121  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1122  EVT MemoryVT = LD->getMemoryVT();
1123  unsigned Alignment = LD->getOriginalAlignment();
1124  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1125  AAMDNodes AAInfo = LD->getAAInfo();
1126 
1127  EVT LoMemVT, HiMemVT;
1128  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1129 
1130  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1131  LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1132 
1133  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1134  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1135  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
1136  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1137  LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1138  Alignment, MMOFlags, AAInfo);
1139 
1140  // Build a factor node to remember that this load is independent of the
1141  // other one.
1142  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1143  Hi.getValue(1));
1144 
1145  // Legalize the chain result - switch anything that used the old chain to
1146  // use the new one.
1147  ReplaceValueWith(SDValue(LD, 1), Ch);
1148 }
1149 
1150 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1151  SDValue &Lo, SDValue &Hi) {
1152  EVT LoVT, HiVT;
1153  SDLoc dl(MLD);
1154  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1155 
1156  SDValue Ch = MLD->getChain();
1157  SDValue Ptr = MLD->getBasePtr();
1158  SDValue Mask = MLD->getMask();
1159  SDValue Src0 = MLD->getSrc0();
1160  unsigned Alignment = MLD->getOriginalAlignment();
1161  ISD::LoadExtType ExtType = MLD->getExtensionType();
1162 
1163  // if Alignment is equal to the vector size,
1164  // take the half of it for the second part
1165  unsigned SecondHalfAlignment =
1166  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1167  Alignment/2 : Alignment;
1168 
1169  // Split Mask operand
1170  SDValue MaskLo, MaskHi;
1171  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1172  GetSplitVector(Mask, MaskLo, MaskHi);
1173  else
1174  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1175 
1176  EVT MemoryVT = MLD->getMemoryVT();
1177  EVT LoMemVT, HiMemVT;
1178  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1179 
1180  SDValue Src0Lo, Src0Hi;
1181  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1182  GetSplitVector(Src0, Src0Lo, Src0Hi);
1183  else
1184  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1185 
1187  getMachineMemOperand(MLD->getPointerInfo(),
1189  Alignment, MLD->getAAInfo(), MLD->getRanges());
1190 
1191  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
1192  ExtType, MLD->isExpandingLoad());
1193 
1194  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1195  MLD->isExpandingLoad());
1196 
1197  MMO = DAG.getMachineFunction().
1198  getMachineMemOperand(MLD->getPointerInfo(),
1200  SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());
1201 
1202  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
1203  ExtType, MLD->isExpandingLoad());
1204 
1205 
1206  // Build a factor node to remember that this load is independent of the
1207  // other one.
1208  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1209  Hi.getValue(1));
1210 
1211  // Legalize the chain result - switch anything that used the old chain to
1212  // use the new one.
1213  ReplaceValueWith(SDValue(MLD, 1), Ch);
1214 
1215 }
1216 
1217 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1218  SDValue &Lo, SDValue &Hi) {
1219  EVT LoVT, HiVT;
1220  SDLoc dl(MGT);
1221  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1222 
1223  SDValue Ch = MGT->getChain();
1224  SDValue Ptr = MGT->getBasePtr();
1225  SDValue Mask = MGT->getMask();
1226  SDValue Src0 = MGT->getValue();
1227  SDValue Index = MGT->getIndex();
1228  unsigned Alignment = MGT->getOriginalAlignment();
1229 
1230  // Split Mask operand
1231  SDValue MaskLo, MaskHi;
1232  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1233  GetSplitVector(Mask, MaskLo, MaskHi);
1234  else
1235  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1236 
1237  EVT MemoryVT = MGT->getMemoryVT();
1238  EVT LoMemVT, HiMemVT;
1239  // Split MemoryVT
1240  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1241 
1242  SDValue Src0Lo, Src0Hi;
1243  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1244  GetSplitVector(Src0, Src0Lo, Src0Hi);
1245  else
1246  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1247 
1248  SDValue IndexHi, IndexLo;
1249  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1250  GetSplitVector(Index, IndexLo, IndexHi);
1251  else
1252  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1253 
1255  getMachineMemOperand(MGT->getPointerInfo(),
1257  Alignment, MGT->getAAInfo(), MGT->getRanges());
1258 
1259  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
1260  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1261  MMO);
1262 
1263  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
1264  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1265  MMO);
1266 
1267  // Build a factor node to remember that this load is independent of the
1268  // other one.
1269  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1270  Hi.getValue(1));
1271 
1272  // Legalize the chain result - switch anything that used the old chain to
1273  // use the new one.
1274  ReplaceValueWith(SDValue(MGT, 1), Ch);
1275 }
1276 
1277 
1278 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1279  assert(N->getValueType(0).isVector() &&
1280  N->getOperand(0).getValueType().isVector() &&
1281  "Operand types must be vectors");
1282 
1283  EVT LoVT, HiVT;
1284  SDLoc DL(N);
1285  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1286 
1287  // Split the input.
1288  SDValue LL, LH, RL, RH;
1289  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1290  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1291 
1292  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1293  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1294 }
1295 
1296 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1297  SDValue &Hi) {
1298  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1299  EVT LoVT, HiVT;
1300  SDLoc dl(N);
1301  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1302 
1303  // If the input also splits, handle it directly for a compile time speedup.
1304  // Otherwise split it by hand.
1305  EVT InVT = N->getOperand(0).getValueType();
1306  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1307  GetSplitVector(N->getOperand(0), Lo, Hi);
1308  else
1309  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1310 
1311  if (N->getOpcode() == ISD::FP_ROUND) {
1312  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1313  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1314  } else {
1315  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1316  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1317  }
1318 }
1319 
1320 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1321  SDValue &Hi) {
1322  SDLoc dl(N);
1323  EVT SrcVT = N->getOperand(0).getValueType();
1324  EVT DestVT = N->getValueType(0);
1325  EVT LoVT, HiVT;
1326  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1327 
1328  // We can do better than a generic split operation if the extend is doing
1329  // more than just doubling the width of the elements and the following are
1330  // true:
1331  // - The number of vector elements is even,
1332  // - the source type is legal,
1333  // - the type of a split source is illegal,
1334  // - the type of an extended (by doubling element size) source is legal, and
1335  // - the type of that extended source when split is legal.
1336  //
1337  // This won't necessarily completely legalize the operation, but it will
1338  // more effectively move in the right direction and prevent falling down
1339  // to scalarization in many cases due to the input vector being split too
1340  // far.
1341  unsigned NumElements = SrcVT.getVectorNumElements();
1342  if ((NumElements & 1) == 0 &&
1343  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1344  LLVMContext &Ctx = *DAG.getContext();
1345  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1346  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1347 
1348  EVT SplitLoVT, SplitHiVT;
1349  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1350  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1351  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1352  DEBUG(dbgs() << "Split vector extend via incremental extend:";
1353  N->dump(&DAG); dbgs() << "\n");
1354  // Extend the source vector by one step.
1355  SDValue NewSrc =
1356  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1357  // Get the low and high halves of the new, extended one step, vector.
1358  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1359  // Extend those vector halves the rest of the way.
1360  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1361  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1362  return;
1363  }
1364  }
1365  // Fall back to the generic unary operator splitting otherwise.
1366  SplitVecRes_UnaryOp(N, Lo, Hi);
1367 }
1368 
1369 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1370  SDValue &Lo, SDValue &Hi) {
1371  // The low and high parts of the original input give four input vectors.
1372  SDValue Inputs[4];
1373  SDLoc dl(N);
1374  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1375  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1376  EVT NewVT = Inputs[0].getValueType();
1377  unsigned NewElts = NewVT.getVectorNumElements();
1378 
1379  // If Lo or Hi uses elements from at most two of the four input vectors, then
1380  // express it as a vector shuffle of those two inputs. Otherwise extract the
1381  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1383  for (unsigned High = 0; High < 2; ++High) {
1384  SDValue &Output = High ? Hi : Lo;
1385 
1386  // Build a shuffle mask for the output, discovering on the fly which
1387  // input vectors to use as shuffle operands (recorded in InputUsed).
1388  // If building a suitable shuffle vector proves too hard, then bail
1389  // out with useBuildVector set.
1390  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1391  unsigned FirstMaskIdx = High * NewElts;
1392  bool useBuildVector = false;
1393  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1394  // The mask element. This indexes into the input.
1395  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1396 
1397  // The input vector this mask element indexes into.
1398  unsigned Input = (unsigned)Idx / NewElts;
1399 
1400  if (Input >= array_lengthof(Inputs)) {
1401  // The mask element does not index into any input vector.
1402  Ops.push_back(-1);
1403  continue;
1404  }
1405 
1406  // Turn the index into an offset from the start of the input vector.
1407  Idx -= Input * NewElts;
1408 
1409  // Find or create a shuffle vector operand to hold this input.
1410  unsigned OpNo;
1411  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1412  if (InputUsed[OpNo] == Input) {
1413  // This input vector is already an operand.
1414  break;
1415  } else if (InputUsed[OpNo] == -1U) {
1416  // Create a new operand for this input vector.
1417  InputUsed[OpNo] = Input;
1418  break;
1419  }
1420  }
1421 
1422  if (OpNo >= array_lengthof(InputUsed)) {
1423  // More than two input vectors used! Give up on trying to create a
1424  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1425  useBuildVector = true;
1426  break;
1427  }
1428 
1429  // Add the mask index for the new shuffle vector.
1430  Ops.push_back(Idx + OpNo * NewElts);
1431  }
1432 
1433  if (useBuildVector) {
1434  EVT EltVT = NewVT.getVectorElementType();
1436 
1437  // Extract the input elements by hand.
1438  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1439  // The mask element. This indexes into the input.
1440  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1441 
1442  // The input vector this mask element indexes into.
1443  unsigned Input = (unsigned)Idx / NewElts;
1444 
1445  if (Input >= array_lengthof(Inputs)) {
1446  // The mask element is "undef" or indexes off the end of the input.
1447  SVOps.push_back(DAG.getUNDEF(EltVT));
1448  continue;
1449  }
1450 
1451  // Turn the index into an offset from the start of the input vector.
1452  Idx -= Input * NewElts;
1453 
1454  // Extract the vector element by hand.
1455  SVOps.push_back(DAG.getNode(
1456  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1457  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1458  }
1459 
1460  // Construct the Lo/Hi output using a BUILD_VECTOR.
1461  Output = DAG.getBuildVector(NewVT, dl, SVOps);
1462  } else if (InputUsed[0] == -1U) {
1463  // No input vectors were used! The result is undefined.
1464  Output = DAG.getUNDEF(NewVT);
1465  } else {
1466  SDValue Op0 = Inputs[InputUsed[0]];
1467  // If only one input was used, use an undefined vector for the other.
1468  SDValue Op1 = InputUsed[1] == -1U ?
1469  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1470  // At least one input vector was used. Create a new shuffle vector.
1471  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1472  }
1473 
1474  Ops.clear();
1475  }
1476 }
1477 
1478 
1479 //===----------------------------------------------------------------------===//
1480 // Operand Vector Splitting
1481 //===----------------------------------------------------------------------===//
1482 
1483 /// This method is called when the specified operand of the specified node is
1484 /// found to need vector splitting. At this point, all of the result types of
1485 /// the node are known to be legal, but other operands of the node may need
1486 /// legalization as well as the specified one.
1487 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1488  DEBUG(dbgs() << "Split node operand: ";
1489  N->dump(&DAG);
1490  dbgs() << "\n");
1491  SDValue Res = SDValue();
1492 
1493  // See if the target wants to custom split this node.
1494  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1495  return false;
1496 
1497  if (!Res.getNode()) {
1498  switch (N->getOpcode()) {
1499  default:
1500 #ifndef NDEBUG
1501  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1502  N->dump(&DAG);
1503  dbgs() << "\n";
1504 #endif
1505  report_fatal_error("Do not know how to split this operator's "
1506  "operand!\n");
1507 
1508  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1509  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1510  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1511  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1512  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1513  case ISD::TRUNCATE:
1514  Res = SplitVecOp_TruncateHelper(N);
1515  break;
1516  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1517  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1518  case ISD::STORE:
1519  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1520  break;
1521  case ISD::MSTORE:
1522  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1523  break;
1524  case ISD::MSCATTER:
1525  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1526  break;
1527  case ISD::MGATHER:
1528  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1529  break;
1530  case ISD::VSELECT:
1531  Res = SplitVecOp_VSELECT(N, OpNo);
1532  break;
1533  case ISD::FP_TO_SINT:
1534  case ISD::FP_TO_UINT:
1535  if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
1536  Res = SplitVecOp_TruncateHelper(N);
1537  else
1538  Res = SplitVecOp_UnaryOp(N);
1539  break;
1540  case ISD::SINT_TO_FP:
1541  case ISD::UINT_TO_FP:
1542  if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
1543  Res = SplitVecOp_TruncateHelper(N);
1544  else
1545  Res = SplitVecOp_UnaryOp(N);
1546  break;
1547  case ISD::CTTZ:
1548  case ISD::CTLZ:
1549  case ISD::CTPOP:
1550  case ISD::FP_EXTEND:
1551  case ISD::SIGN_EXTEND:
1552  case ISD::ZERO_EXTEND:
1553  case ISD::ANY_EXTEND:
1554  case ISD::FTRUNC:
1555  case ISD::FCANONICALIZE:
1556  Res = SplitVecOp_UnaryOp(N);
1557  break;
1558 
1562  Res = SplitVecOp_ExtVecInRegOp(N);
1563  break;
1564 
1565  case ISD::VECREDUCE_FADD:
1566  case ISD::VECREDUCE_FMUL:
1567  case ISD::VECREDUCE_ADD:
1568  case ISD::VECREDUCE_MUL:
1569  case ISD::VECREDUCE_AND:
1570  case ISD::VECREDUCE_OR:
1571  case ISD::VECREDUCE_XOR:
1572  case ISD::VECREDUCE_SMAX:
1573  case ISD::VECREDUCE_SMIN:
1574  case ISD::VECREDUCE_UMAX:
1575  case ISD::VECREDUCE_UMIN:
1576  case ISD::VECREDUCE_FMAX:
1577  case ISD::VECREDUCE_FMIN:
1578  Res = SplitVecOp_VECREDUCE(N, OpNo);
1579  break;
1580  }
1581  }
1582 
1583  // If the result is null, the sub-method took care of registering results etc.
1584  if (!Res.getNode()) return false;
1585 
1586  // If the result is N, the sub-method updated N in place. Tell the legalizer
1587  // core about this.
1588  if (Res.getNode() == N)
1589  return true;
1590 
1591  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1592  "Invalid operand expansion");
1593 
1594  ReplaceValueWith(SDValue(N, 0), Res);
1595  return false;
1596 }
1597 
1598 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1599  // The only possibility for an illegal operand is the mask, since result type
1600  // legalization would have handled this node already otherwise.
1601  assert(OpNo == 0 && "Illegal operand must be mask");
1602 
1603  SDValue Mask = N->getOperand(0);
1604  SDValue Src0 = N->getOperand(1);
1605  SDValue Src1 = N->getOperand(2);
1606  EVT Src0VT = Src0.getValueType();
1607  SDLoc DL(N);
1608  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1609 
1610  SDValue Lo, Hi;
1611  GetSplitVector(N->getOperand(0), Lo, Hi);
1612  assert(Lo.getValueType() == Hi.getValueType() &&
1613  "Lo and Hi have differing types");
1614 
1615  EVT LoOpVT, HiOpVT;
1616  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1617  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1618 
1619  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1620  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1621  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1622  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1623 
1624  SDValue LoSelect =
1625  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1626  SDValue HiSelect =
1627  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1628 
1629  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1630 }
1631 
1632 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
1633  EVT ResVT = N->getValueType(0);
1634  SDValue Lo, Hi;
1635  SDLoc dl(N);
1636 
1637  SDValue VecOp = N->getOperand(OpNo);
1638  EVT VecVT = VecOp.getValueType();
1639  assert(VecVT.isVector() && "Can only split reduce vector operand");
1640  GetSplitVector(VecOp, Lo, Hi);
1641  EVT LoOpVT, HiOpVT;
1642  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
1643 
1644  bool NoNaN = N->getFlags().hasNoNaNs();
1645  unsigned CombineOpc = 0;
1646  switch (N->getOpcode()) {
1647  case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
1648  case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
1649  case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
1650  case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
1651  case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
1652  case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
1653  case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
1654  case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
1655  case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
1656  case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
1657  case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
1658  case ISD::VECREDUCE_FMAX:
1659  CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
1660  break;
1661  case ISD::VECREDUCE_FMIN:
1662  CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
1663  break;
1664  default:
1665  llvm_unreachable("Unexpected reduce ISD node");
1666  }
1667 
1668  // Use the appropriate scalar instruction on the split subvectors before
1669  // reducing the now partially reduced smaller vector.
1670  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi);
1671  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial);
1672 }
1673 
1674 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1675  // The result has a legal vector type, but the input needs splitting.
1676  EVT ResVT = N->getValueType(0);
1677  SDValue Lo, Hi;
1678  SDLoc dl(N);
1679  GetSplitVector(N->getOperand(0), Lo, Hi);
1680  EVT InVT = Lo.getValueType();
1681 
1682  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1683  InVT.getVectorNumElements());
1684 
1685  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1686  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1687 
1688  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1689 }
1690 
1691 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1692  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1693  // end up being split all the way down to individual components. Convert the
1694  // split pieces into integers and reassemble.
1695  SDValue Lo, Hi;
1696  GetSplitVector(N->getOperand(0), Lo, Hi);
1697  Lo = BitConvertToInteger(Lo);
1698  Hi = BitConvertToInteger(Hi);
1699 
1700  if (DAG.getDataLayout().isBigEndian())
1701  std::swap(Lo, Hi);
1702 
1703  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1704  JoinIntegers(Lo, Hi));
1705 }
1706 
1707 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1708  // We know that the extracted result type is legal.
1709  EVT SubVT = N->getValueType(0);
1710  SDValue Idx = N->getOperand(1);
1711  SDLoc dl(N);
1712  SDValue Lo, Hi;
1713  GetSplitVector(N->getOperand(0), Lo, Hi);
1714 
1715  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1716  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1717 
1718  if (IdxVal < LoElts) {
1719  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1720  "Extracted subvector crosses vector split!");
1721  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1722  } else {
1723  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1724  DAG.getConstant(IdxVal - LoElts, dl,
1725  Idx.getValueType()));
1726  }
1727 }
1728 
1729 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1730  SDValue Vec = N->getOperand(0);
1731  SDValue Idx = N->getOperand(1);
1732  EVT VecVT = Vec.getValueType();
1733 
1734  if (isa<ConstantSDNode>(Idx)) {
1735  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1736  assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1737 
1738  SDValue Lo, Hi;
1739  GetSplitVector(Vec, Lo, Hi);
1740 
1741  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1742 
1743  if (IdxVal < LoElts)
1744  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1745  return SDValue(DAG.UpdateNodeOperands(N, Hi,
1746  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
1747  Idx.getValueType())), 0);
1748  }
1749 
1750  // See if the target wants to custom expand this node.
1751  if (CustomLowerNode(N, N->getValueType(0), true))
1752  return SDValue();
1753 
1754  // Make the vector elements byte-addressable if they aren't already.
1755  SDLoc dl(N);
1756  EVT EltVT = VecVT.getVectorElementType();
1757  if (EltVT.getSizeInBits() < 8) {
1758  SmallVector<SDValue, 4> ElementOps;
1759  for (unsigned i = 0; i < VecVT.getVectorNumElements(); ++i) {
1760  ElementOps.push_back(DAG.getAnyExtOrTrunc(
1761  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vec,
1762  DAG.getConstant(i, dl, MVT::i8)),
1763  dl, MVT::i8));
1764  }
1765 
1766  EltVT = MVT::i8;
1767  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1768  VecVT.getVectorNumElements());
1769  Vec = DAG.getBuildVector(VecVT, dl, ElementOps);
1770  }
1771 
1772  // Store the vector to the stack.
1773  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1774  SDValue Store =
1775  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1776 
1777  // Load back the required element.
1778  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1779  return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1780  MachinePointerInfo(), EltVT);
1781 }
1782 
1783 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
1784  SDValue Lo, Hi;
1785 
1786  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
1787  // splitting the result has the same effect as splitting the input operand.
1788  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1789 
1790  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
1791 }
1792 
1793 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
1794  unsigned OpNo) {
1795  EVT LoVT, HiVT;
1796  SDLoc dl(MGT);
1797  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1798 
1799  SDValue Ch = MGT->getChain();
1800  SDValue Ptr = MGT->getBasePtr();
1801  SDValue Index = MGT->getIndex();
1802  SDValue Mask = MGT->getMask();
1803  SDValue Src0 = MGT->getValue();
1804  unsigned Alignment = MGT->getOriginalAlignment();
1805 
1806  SDValue MaskLo, MaskHi;
1807  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1808  // Split Mask operand
1809  GetSplitVector(Mask, MaskLo, MaskHi);
1810  else
1811  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1812 
1813  EVT MemoryVT = MGT->getMemoryVT();
1814  EVT LoMemVT, HiMemVT;
1815  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1816 
1817  SDValue Src0Lo, Src0Hi;
1818  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1819  GetSplitVector(Src0, Src0Lo, Src0Hi);
1820  else
1821  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1822 
1823  SDValue IndexHi, IndexLo;
1824  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1825  GetSplitVector(Index, IndexLo, IndexHi);
1826  else
1827  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1828 
1830  getMachineMemOperand(MGT->getPointerInfo(),
1832  Alignment, MGT->getAAInfo(), MGT->getRanges());
1833 
1834  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
1835  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
1836  OpsLo, MMO);
1837 
1838  MMO = DAG.getMachineFunction().
1839  getMachineMemOperand(MGT->getPointerInfo(),
1841  Alignment, MGT->getAAInfo(),
1842  MGT->getRanges());
1843 
1844  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
1845  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
1846  OpsHi, MMO);
1847 
1848  // Build a factor node to remember that this load is independent of the
1849  // other one.
1850  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1851  Hi.getValue(1));
1852 
1853  // Legalize the chain result - switch anything that used the old chain to
1854  // use the new one.
1855  ReplaceValueWith(SDValue(MGT, 1), Ch);
1856 
1857  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
1858  Hi);
1859  ReplaceValueWith(SDValue(MGT, 0), Res);
1860  return SDValue();
1861 }
1862 
1863 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
1864  unsigned OpNo) {
1865  SDValue Ch = N->getChain();
1866  SDValue Ptr = N->getBasePtr();
1867  SDValue Mask = N->getMask();
1868  SDValue Data = N->getValue();
1869  EVT MemoryVT = N->getMemoryVT();
1870  unsigned Alignment = N->getOriginalAlignment();
1871  SDLoc DL(N);
1872 
1873  EVT LoMemVT, HiMemVT;
1874  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1875 
1876  SDValue DataLo, DataHi;
1877  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1878  // Split Data operand
1879  GetSplitVector(Data, DataLo, DataHi);
1880  else
1881  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1882 
1883  SDValue MaskLo, MaskHi;
1884  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1885  // Split Mask operand
1886  GetSplitVector(Mask, MaskLo, MaskHi);
1887  else
1888  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1889 
1890  MaskLo = PromoteTargetBoolean(MaskLo, DataLo.getValueType());
1891  MaskHi = PromoteTargetBoolean(MaskHi, DataHi.getValueType());
1892 
1893  // if Alignment is equal to the vector size,
1894  // take the half of it for the second part
1895  unsigned SecondHalfAlignment =
1896  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
1897  Alignment/2 : Alignment;
1898 
1899  SDValue Lo, Hi;
1901  getMachineMemOperand(N->getPointerInfo(),
1903  Alignment, N->getAAInfo(), N->getRanges());
1904 
1905  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
1906  N->isTruncatingStore(),
1907  N->isCompressingStore());
1908 
1909  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
1910  N->isCompressingStore());
1911  MMO = DAG.getMachineFunction().
1912  getMachineMemOperand(N->getPointerInfo(),
1914  SecondHalfAlignment, N->getAAInfo(), N->getRanges());
1915 
1916  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
1918 
1919  // Build a factor node to remember that this store is independent of the
1920  // other one.
1921  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1922 }
1923 
1924 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
1925  unsigned OpNo) {
1926  SDValue Ch = N->getChain();
1927  SDValue Ptr = N->getBasePtr();
1928  SDValue Mask = N->getMask();
1929  SDValue Index = N->getIndex();
1930  SDValue Data = N->getValue();
1931  EVT MemoryVT = N->getMemoryVT();
1932  unsigned Alignment = N->getOriginalAlignment();
1933  SDLoc DL(N);
1934 
1935  // Split all operands
1936  EVT LoMemVT, HiMemVT;
1937  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1938 
1939  SDValue DataLo, DataHi;
1940  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1941  // Split Data operand
1942  GetSplitVector(Data, DataLo, DataHi);
1943  else
1944  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1945 
1946  SDValue MaskLo, MaskHi;
1947  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1948  // Split Mask operand
1949  GetSplitVector(Mask, MaskLo, MaskHi);
1950  else
1951  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1952 
1953  SDValue IndexHi, IndexLo;
1954  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1955  GetSplitVector(Index, IndexLo, IndexHi);
1956  else
1957  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
1958 
1959  SDValue Lo;
1961  getMachineMemOperand(N->getPointerInfo(),
1963  Alignment, N->getAAInfo(), N->getRanges());
1964 
1965  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo};
1966  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
1967  DL, OpsLo, MMO);
1968 
1969  MMO = DAG.getMachineFunction().
1970  getMachineMemOperand(N->getPointerInfo(),
1972  Alignment, N->getAAInfo(), N->getRanges());
1973 
1974  // The order of the Scatter operation after split is well defined. The "Hi"
1975  // part comes after the "Lo". So these two operations should be chained one
1976  // after another.
1977  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi};
1978  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
1979  DL, OpsHi, MMO);
1980 }
1981 
1982 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
1983  assert(N->isUnindexed() && "Indexed store of vector?");
1984  assert(OpNo == 1 && "Can only split the stored value");
1985  SDLoc DL(N);
1986 
1987  bool isTruncating = N->isTruncatingStore();
1988  SDValue Ch = N->getChain();
1989  SDValue Ptr = N->getBasePtr();
1990  EVT MemoryVT = N->getMemoryVT();
1991  unsigned Alignment = N->getOriginalAlignment();
1992  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
1993  AAMDNodes AAInfo = N->getAAInfo();
1994  SDValue Lo, Hi;
1995  GetSplitVector(N->getOperand(1), Lo, Hi);
1996 
1997  EVT LoMemVT, HiMemVT;
1998  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1999 
2000  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2001 
2002  if (isTruncating)
2003  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2004  Alignment, MMOFlags, AAInfo);
2005  else
2006  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2007  AAInfo);
2008 
2009  // Increment the pointer to the other half.
2010  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
2011  DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
2012 
2013  if (isTruncating)
2014  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2015  N->getPointerInfo().getWithOffset(IncrementSize),
2016  HiMemVT, Alignment, MMOFlags, AAInfo);
2017  else
2018  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2019  N->getPointerInfo().getWithOffset(IncrementSize),
2020  Alignment, MMOFlags, AAInfo);
2021 
2022  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2023 }
2024 
2025 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2026  SDLoc DL(N);
2027 
2028  // The input operands all must have the same type, and we know the result
2029  // type is valid. Convert this to a buildvector which extracts all the
2030  // input elements.
2031  // TODO: If the input elements are power-two vectors, we could convert this to
2032  // a new CONCAT_VECTORS node with elements that are half-wide.
2034  EVT EltVT = N->getValueType(0).getVectorElementType();
2035  for (const SDValue &Op : N->op_values()) {
2036  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2037  i != e; ++i) {
2038  Elts.push_back(DAG.getNode(
2039  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2040  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2041  }
2042  }
2043 
2044  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2045 }
2046 
2047 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2048  // The result type is legal, but the input type is illegal. If splitting
2049  // ends up with the result type of each half still being legal, just
2050  // do that. If, however, that would result in an illegal result type,
2051  // we can try to get more clever with power-two vectors. Specifically,
2052  // split the input type, but also widen the result element size, then
2053  // concatenate the halves and truncate again. For example, consider a target
2054  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2055  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2056  // %inlo = v4i32 extract_subvector %in, 0
2057  // %inhi = v4i32 extract_subvector %in, 4
2058  // %lo16 = v4i16 trunc v4i32 %inlo
2059  // %hi16 = v4i16 trunc v4i32 %inhi
2060  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2061  // %res = v8i8 trunc v8i16 %in16
2062  //
2063  // Without this transform, the original truncate would end up being
2064  // scalarized, which is pretty much always a last resort.
2065  SDValue InVec = N->getOperand(0);
2066  EVT InVT = InVec->getValueType(0);
2067  EVT OutVT = N->getValueType(0);
2068  unsigned NumElements = OutVT.getVectorNumElements();
2069  bool IsFloat = OutVT.isFloatingPoint();
2070 
2071  // Widening should have already made sure this is a power-two vector
2072  // if we're trying to split it at all. assert() that's true, just in case.
2073  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2074 
2075  unsigned InElementSize = InVT.getScalarSizeInBits();
2076  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2077 
2078  // If the input elements are only 1/2 the width of the result elements,
2079  // just use the normal splitting. Our trick only work if there's room
2080  // to split more than once.
2081  if (InElementSize <= OutElementSize * 2)
2082  return SplitVecOp_UnaryOp(N);
2083  SDLoc DL(N);
2084 
2085  // Extract the halves of the input via extract_subvector.
2086  SDValue InLoVec, InHiVec;
2087  std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL);
2088  // Truncate them to 1/2 the element size.
2089  EVT HalfElementVT = IsFloat ?
2090  EVT::getFloatingPointVT(InElementSize/2) :
2091  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2092  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2093  NumElements/2);
2094  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2095  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2096  // Concatenate them to get the full intermediate truncation result.
2097  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2098  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2099  HalfHi);
2100  // Now finish up by truncating all the way down to the original result
2101  // type. This should normally be something that ends up being legal directly,
2102  // but in theory if a target has very wide vectors and an annoyingly
2103  // restricted set of legal types, this split can chain to build things up.
2104  return IsFloat
2105  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2106  DAG.getTargetConstant(
2107  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2108  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2109 }
2110 
2111 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2112  assert(N->getValueType(0).isVector() &&
2113  N->getOperand(0).getValueType().isVector() &&
2114  "Operand types must be vectors");
2115  // The result has a legal vector type, but the input needs splitting.
2116  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2117  SDLoc DL(N);
2118  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2119  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2120  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2121  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2122  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2123 
2124  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2125  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2126  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2127  return PromoteTargetBoolean(Con, N->getValueType(0));
2128 }
2129 
2130 
2131 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2132  // The result has a legal vector type, but the input needs splitting.
2133  EVT ResVT = N->getValueType(0);
2134  SDValue Lo, Hi;
2135  SDLoc DL(N);
2136  GetSplitVector(N->getOperand(0), Lo, Hi);
2137  EVT InVT = Lo.getValueType();
2138 
2139  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2140  InVT.getVectorNumElements());
2141 
2142  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2143  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2144 
2145  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2146 }
2147 
2148 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2149  // The result (and the first input) has a legal vector type, but the second
2150  // input needs splitting.
2151  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2152 }
2153 
2154 
2155 //===----------------------------------------------------------------------===//
2156 // Result Vector Widening
2157 //===----------------------------------------------------------------------===//
2158 
2159 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2160  DEBUG(dbgs() << "Widen node result " << ResNo << ": ";
2161  N->dump(&DAG);
2162  dbgs() << "\n");
2163 
2164  // See if the target wants to custom widen this node.
2165  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2166  return;
2167 
2168  SDValue Res = SDValue();
2169  switch (N->getOpcode()) {
2170  default:
2171 #ifndef NDEBUG
2172  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2173  N->dump(&DAG);
2174  dbgs() << "\n";
2175 #endif
2176  llvm_unreachable("Do not know how to widen the result of this operator!");
2177 
2178  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2179  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2180  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2181  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2182  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2183  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2184  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2185  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2186  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2187  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2188  case ISD::VSELECT:
2189  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2190  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2191  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2192  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2193  case ISD::VECTOR_SHUFFLE:
2194  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2195  break;
2196  case ISD::MLOAD:
2197  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2198  break;
2199  case ISD::MGATHER:
2200  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2201  break;
2202 
2203  case ISD::ADD:
2204  case ISD::AND:
2205  case ISD::MUL:
2206  case ISD::MULHS:
2207  case ISD::MULHU:
2208  case ISD::OR:
2209  case ISD::SUB:
2210  case ISD::XOR:
2211  case ISD::FMINNUM:
2212  case ISD::FMAXNUM:
2213  case ISD::FMINNAN:
2214  case ISD::FMAXNAN:
2215  case ISD::SMIN:
2216  case ISD::SMAX:
2217  case ISD::UMIN:
2218  case ISD::UMAX:
2219  Res = WidenVecRes_Binary(N);
2220  break;
2221 
2222  case ISD::FADD:
2223  case ISD::FMUL:
2224  case ISD::FPOW:
2225  case ISD::FSUB:
2226  case ISD::FDIV:
2227  case ISD::FREM:
2228  case ISD::SDIV:
2229  case ISD::UDIV:
2230  case ISD::SREM:
2231  case ISD::UREM:
2232  Res = WidenVecRes_BinaryCanTrap(N);
2233  break;
2234 
2235  case ISD::FCOPYSIGN:
2236  Res = WidenVecRes_FCOPYSIGN(N);
2237  break;
2238 
2239  case ISD::FPOWI:
2240  Res = WidenVecRes_POWI(N);
2241  break;
2242 
2243  case ISD::SHL:
2244  case ISD::SRA:
2245  case ISD::SRL:
2246  Res = WidenVecRes_Shift(N);
2247  break;
2248 
2252  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2253  break;
2254 
2255  case ISD::ANY_EXTEND:
2256  case ISD::FP_EXTEND:
2257  case ISD::FP_ROUND:
2258  case ISD::FP_TO_SINT:
2259  case ISD::FP_TO_UINT:
2260  case ISD::SIGN_EXTEND:
2261  case ISD::SINT_TO_FP:
2262  case ISD::TRUNCATE:
2263  case ISD::UINT_TO_FP:
2264  case ISD::ZERO_EXTEND:
2265  Res = WidenVecRes_Convert(N);
2266  break;
2267 
2268  case ISD::BITREVERSE:
2269  case ISD::BSWAP:
2270  case ISD::CTLZ:
2271  case ISD::CTPOP:
2272  case ISD::CTTZ:
2273  case ISD::FABS:
2274  case ISD::FCEIL:
2275  case ISD::FCOS:
2276  case ISD::FEXP:
2277  case ISD::FEXP2:
2278  case ISD::FFLOOR:
2279  case ISD::FLOG:
2280  case ISD::FLOG10:
2281  case ISD::FLOG2:
2282  case ISD::FNEARBYINT:
2283  case ISD::FNEG:
2284  case ISD::FRINT:
2285  case ISD::FROUND:
2286  case ISD::FSIN:
2287  case ISD::FSQRT:
2288  case ISD::FTRUNC:
2289  Res = WidenVecRes_Unary(N);
2290  break;
2291  case ISD::FMA:
2292  Res = WidenVecRes_Ternary(N);
2293  break;
2294  }
2295 
2296  // If Res is null, the sub-method took care of registering the result.
2297  if (Res.getNode())
2298  SetWidenedVector(SDValue(N, ResNo), Res);
2299 }
2300 
2301 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2302  // Ternary op widening.
2303  SDLoc dl(N);
2304  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2305  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2306  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2307  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2308  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2309 }
2310 
2311 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2312  // Binary op widening.
2313  SDLoc dl(N);
2314  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2315  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2316  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2317  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2318 }
2319 
2320 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2321  // Binary op widening for operations that can trap.
2322  unsigned Opcode = N->getOpcode();
2323  SDLoc dl(N);
2324  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2325  EVT WidenEltVT = WidenVT.getVectorElementType();
2326  EVT VT = WidenVT;
2327  unsigned NumElts = VT.getVectorNumElements();
2328  const SDNodeFlags Flags = N->getFlags();
2329  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2330  NumElts = NumElts / 2;
2331  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2332  }
2333 
2334  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2335  // Operation doesn't trap so just widen as normal.
2336  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2337  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2338  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2339  }
2340 
2341  // No legal vector version so unroll the vector operation and then widen.
2342  if (NumElts == 1)
2343  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2344 
2345  // Since the operation can trap, apply operation on the original vector.
2346  EVT MaxVT = VT;
2347  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2348  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2349  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2350 
2351  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2352  unsigned ConcatEnd = 0; // Current ConcatOps index.
2353  int Idx = 0; // Current Idx into input vectors.
2354 
2355  // NumElts := greatest legal vector size (at most WidenVT)
2356  // while (orig. vector has unhandled elements) {
2357  // take munches of size NumElts from the beginning and add to ConcatOps
2358  // NumElts := next smaller supported vector size or 1
2359  // }
2360  while (CurNumElts != 0) {
2361  while (CurNumElts >= NumElts) {
2362  SDValue EOp1 = DAG.getNode(
2363  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2364  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2365  SDValue EOp2 = DAG.getNode(
2366  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2367  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2368  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2369  Idx += NumElts;
2370  CurNumElts -= NumElts;
2371  }
2372  do {
2373  NumElts = NumElts / 2;
2374  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2375  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2376 
2377  if (NumElts == 1) {
2378  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2379  SDValue EOp1 = DAG.getNode(
2380  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2381  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2382  SDValue EOp2 = DAG.getNode(
2383  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2384  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2385  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2386  EOp1, EOp2, Flags);
2387  }
2388  CurNumElts = 0;
2389  }
2390  }
2391 
2392  // Check to see if we have a single operation with the widen type.
2393  if (ConcatEnd == 1) {
2394  VT = ConcatOps[0].getValueType();
2395  if (VT == WidenVT)
2396  return ConcatOps[0];
2397  }
2398 
2399  // while (Some element of ConcatOps is not of type MaxVT) {
2400  // From the end of ConcatOps, collect elements of the same type and put
2401  // them into an op of the next larger supported type
2402  // }
2403  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2404  Idx = ConcatEnd - 1;
2405  VT = ConcatOps[Idx--].getValueType();
2406  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2407  Idx--;
2408 
2409  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2410  EVT NextVT;
2411  do {
2412  NextSize *= 2;
2413  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2414  } while (!TLI.isTypeLegal(NextVT));
2415 
2416  if (!VT.isVector()) {
2417  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2418  SDValue VecOp = DAG.getUNDEF(NextVT);
2419  unsigned NumToInsert = ConcatEnd - Idx - 1;
2420  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2421  VecOp = DAG.getNode(
2422  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2423  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2424  }
2425  ConcatOps[Idx+1] = VecOp;
2426  ConcatEnd = Idx + 2;
2427  } else {
2428  // Vector type, create a CONCAT_VECTORS of type NextVT
2429  SDValue undefVec = DAG.getUNDEF(VT);
2430  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2431  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2432  unsigned RealVals = ConcatEnd - Idx - 1;
2433  unsigned SubConcatEnd = 0;
2434  unsigned SubConcatIdx = Idx + 1;
2435  while (SubConcatEnd < RealVals)
2436  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2437  while (SubConcatEnd < OpsToConcat)
2438  SubConcatOps[SubConcatEnd++] = undefVec;
2439  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2440  NextVT, SubConcatOps);
2441  ConcatEnd = SubConcatIdx + 1;
2442  }
2443  }
2444 
2445  // Check to see if we have a single operation with the widen type.
2446  if (ConcatEnd == 1) {
2447  VT = ConcatOps[0].getValueType();
2448  if (VT == WidenVT)
2449  return ConcatOps[0];
2450  }
2451 
2452  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2453  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2454  if (NumOps != ConcatEnd ) {
2455  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2456  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2457  ConcatOps[j] = UndefVal;
2458  }
2459  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2460  makeArrayRef(ConcatOps.data(), NumOps));
2461 }
2462 
2463 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
2464  SDValue InOp = N->getOperand(0);
2465  SDLoc DL(N);
2466 
2467  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2468  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2469 
2470  EVT InVT = InOp.getValueType();
2471  EVT InEltVT = InVT.getVectorElementType();
2472  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2473 
2474  unsigned Opcode = N->getOpcode();
2475  unsigned InVTNumElts = InVT.getVectorNumElements();
2476  const SDNodeFlags Flags = N->getFlags();
2477  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2478  InOp = GetWidenedVector(N->getOperand(0));
2479  InVT = InOp.getValueType();
2480  InVTNumElts = InVT.getVectorNumElements();
2481  if (InVTNumElts == WidenNumElts) {
2482  if (N->getNumOperands() == 1)
2483  return DAG.getNode(Opcode, DL, WidenVT, InOp);
2484  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
2485  }
2486  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
2487  // If both input and result vector types are of same width, extend
2488  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
2489  // accepts fewer elements in the result than in the input.
2490  if (Opcode == ISD::SIGN_EXTEND)
2491  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2492  if (Opcode == ISD::ZERO_EXTEND)
2493  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2494  }
2495  }
2496 
2497  if (TLI.isTypeLegal(InWidenVT)) {
2498  // Because the result and the input are different vector types, widening
2499  // the result could create a legal type but widening the input might make
2500  // it an illegal type that might lead to repeatedly splitting the input
2501  // and then widening it. To avoid this, we widen the input only if
2502  // it results in a legal type.
2503  if (WidenNumElts % InVTNumElts == 0) {
2504  // Widen the input and call convert on the widened input vector.
2505  unsigned NumConcat = WidenNumElts/InVTNumElts;
2506  SmallVector<SDValue, 16> Ops(NumConcat);
2507  Ops[0] = InOp;
2508  SDValue UndefVal = DAG.getUNDEF(InVT);
2509  for (unsigned i = 1; i != NumConcat; ++i)
2510  Ops[i] = UndefVal;
2511  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
2512  if (N->getNumOperands() == 1)
2513  return DAG.getNode(Opcode, DL, WidenVT, InVec);
2514  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
2515  }
2516 
2517  if (InVTNumElts % WidenNumElts == 0) {
2518  SDValue InVal = DAG.getNode(
2519  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
2520  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2521  // Extract the input and convert the shorten input vector.
2522  if (N->getNumOperands() == 1)
2523  return DAG.getNode(Opcode, DL, WidenVT, InVal);
2524  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
2525  }
2526  }
2527 
2528  // Otherwise unroll into some nasty scalar code and rebuild the vector.
2529  SmallVector<SDValue, 16> Ops(WidenNumElts);
2530  EVT EltVT = WidenVT.getVectorElementType();
2531  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2532  unsigned i;
2533  for (i=0; i < MinElts; ++i) {
2534  SDValue Val = DAG.getNode(
2535  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
2536  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2537  if (N->getNumOperands() == 1)
2538  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
2539  else
2540  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
2541  }
2542 
2543  SDValue UndefVal = DAG.getUNDEF(EltVT);
2544  for (; i < WidenNumElts; ++i)
2545  Ops[i] = UndefVal;
2546 
2547  return DAG.getBuildVector(WidenVT, DL, Ops);
2548 }
2549 
2550 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
2551  unsigned Opcode = N->getOpcode();
2552  SDValue InOp = N->getOperand(0);
2553  SDLoc DL(N);
2554 
2555  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2556  EVT WidenSVT = WidenVT.getVectorElementType();
2557  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2558 
2559  EVT InVT = InOp.getValueType();
2560  EVT InSVT = InVT.getVectorElementType();
2561  unsigned InVTNumElts = InVT.getVectorNumElements();
2562 
2563  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2564  InOp = GetWidenedVector(InOp);
2565  InVT = InOp.getValueType();
2566  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
2567  switch (Opcode) {
2569  return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
2571  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2573  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2574  }
2575  }
2576  }
2577 
2578  // Unroll, extend the scalars and rebuild the vector.
2580  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
2581  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
2582  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2583  switch (Opcode) {
2585  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
2586  break;
2588  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
2589  break;
2591  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
2592  break;
2593  default:
2594  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
2595  }
2596  Ops.push_back(Val);
2597  }
2598 
2599  while (Ops.size() != WidenNumElts)
2600  Ops.push_back(DAG.getUNDEF(WidenSVT));
2601 
2602  return DAG.getBuildVector(WidenVT, DL, Ops);
2603 }
2604 
2605 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
2606  // If this is an FCOPYSIGN with same input types, we can treat it as a
2607  // normal (can trap) binary op.
2608  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
2609  return WidenVecRes_BinaryCanTrap(N);
2610 
2611  // If the types are different, fall back to unrolling.
2612  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2613  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2614 }
2615 
2616 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
2617  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2618  SDValue InOp = GetWidenedVector(N->getOperand(0));
2619  SDValue ShOp = N->getOperand(1);
2620  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2621 }
2622 
2623 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
2624  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2625  SDValue InOp = GetWidenedVector(N->getOperand(0));
2626  SDValue ShOp = N->getOperand(1);
2627 
2628  EVT ShVT = ShOp.getValueType();
2629  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
2630  ShOp = GetWidenedVector(ShOp);
2631  ShVT = ShOp.getValueType();
2632  }
2633  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
2634  ShVT.getVectorElementType(),
2635  WidenVT.getVectorNumElements());
2636  if (ShVT != ShWidenVT)
2637  ShOp = ModifyToType(ShOp, ShWidenVT);
2638 
2639  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2640 }
2641 
2642 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
2643  // Unary op widening.
2644  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2645  SDValue InOp = GetWidenedVector(N->getOperand(0));
2646  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
2647 }
2648 
2649 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
2650  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2651  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
2652  cast<VTSDNode>(N->getOperand(1))->getVT()
2653  .getVectorElementType(),
2654  WidenVT.getVectorNumElements());
2655  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
2656  return DAG.getNode(N->getOpcode(), SDLoc(N),
2657  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
2658 }
2659 
2660 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
2661  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2662  return GetWidenedVector(WidenVec);
2663 }
2664 
2665 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
2666  SDValue InOp = N->getOperand(0);
2667  EVT InVT = InOp.getValueType();
2668  EVT VT = N->getValueType(0);
2669  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2670  SDLoc dl(N);
2671 
2672  switch (getTypeAction(InVT)) {
2674  break;
2676  // If the incoming type is a vector that is being promoted, then
2677  // we know that the elements are arranged differently and that we
2678  // must perform the conversion using a stack slot.
2679  if (InVT.isVector())
2680  break;
2681 
2682  // If the InOp is promoted to the same size, convert it. Otherwise,
2683  // fall out of the switch and widen the promoted input.
2684  InOp = GetPromotedInteger(InOp);
2685  InVT = InOp.getValueType();
2686  if (WidenVT.bitsEq(InVT))
2687  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2688  break;
2695  break;
2697  // If the InOp is widened to the same size, convert it. Otherwise, fall
2698  // out of the switch and widen the widened input.
2699  InOp = GetWidenedVector(InOp);
2700  InVT = InOp.getValueType();
2701  if (WidenVT.bitsEq(InVT))
2702  // The input widens to the same size. Convert to the widen value.
2703  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2704  break;
2705  }
2706 
2707  unsigned WidenSize = WidenVT.getSizeInBits();
2708  unsigned InSize = InVT.getSizeInBits();
2709  // x86mmx is not an acceptable vector element type, so don't try.
2710  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
2711  // Determine new input vector type. The new input vector type will use
2712  // the same element type (if its a vector) or use the input type as a
2713  // vector. It is the same size as the type to widen to.
2714  EVT NewInVT;
2715  unsigned NewNumElts = WidenSize / InSize;
2716  if (InVT.isVector()) {
2717  EVT InEltVT = InVT.getVectorElementType();
2718  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
2719  WidenSize / InEltVT.getSizeInBits());
2720  } else {
2721  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
2722  }
2723 
2724  if (TLI.isTypeLegal(NewInVT)) {
2725  // Because the result and the input are different vector types, widening
2726  // the result could create a legal type but widening the input might make
2727  // it an illegal type that might lead to repeatedly splitting the input
2728  // and then widening it. To avoid this, we widen the input only if
2729  // it results in a legal type.
2730  SmallVector<SDValue, 16> Ops(NewNumElts);
2731  SDValue UndefVal = DAG.getUNDEF(InVT);
2732  Ops[0] = InOp;
2733  for (unsigned i = 1; i < NewNumElts; ++i)
2734  Ops[i] = UndefVal;
2735 
2736  SDValue NewVec;
2737  if (InVT.isVector())
2738  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
2739  else
2740  NewVec = DAG.getBuildVector(NewInVT, dl, Ops);
2741  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
2742  }
2743  }
2744 
2745  return CreateStackStoreLoad(InOp, WidenVT);
2746 }
2747 
2748 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
2749  SDLoc dl(N);
2750  // Build a vector with undefined for the new nodes.
2751  EVT VT = N->getValueType(0);
2752 
2753  // Integer BUILD_VECTOR operands may be larger than the node's vector element
2754  // type. The UNDEFs need to have the same type as the existing operands.
2755  EVT EltVT = N->getOperand(0).getValueType();
2756  unsigned NumElts = VT.getVectorNumElements();
2757 
2758  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2759  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2760 
2761  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
2762  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
2763  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
2764 
2765  return DAG.getBuildVector(WidenVT, dl, NewOps);
2766 }
2767 
2768 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
2769  EVT InVT = N->getOperand(0).getValueType();
2770  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2771  SDLoc dl(N);
2772  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2773  unsigned NumInElts = InVT.getVectorNumElements();
2774  unsigned NumOperands = N->getNumOperands();
2775 
2776  bool InputWidened = false; // Indicates we need to widen the input.
2777  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
2778  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
2779  // Add undef vectors to widen to correct length.
2780  unsigned NumConcat = WidenVT.getVectorNumElements() /
2781  InVT.getVectorNumElements();
2782  SDValue UndefVal = DAG.getUNDEF(InVT);
2783  SmallVector<SDValue, 16> Ops(NumConcat);
2784  for (unsigned i=0; i < NumOperands; ++i)
2785  Ops[i] = N->getOperand(i);
2786  for (unsigned i = NumOperands; i != NumConcat; ++i)
2787  Ops[i] = UndefVal;
2788  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
2789  }
2790  } else {
2791  InputWidened = true;
2792  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
2793  // The inputs and the result are widen to the same value.
2794  unsigned i;
2795  for (i=1; i < NumOperands; ++i)
2796  if (!N->getOperand(i).isUndef())
2797  break;
2798 
2799  if (i == NumOperands)
2800  // Everything but the first operand is an UNDEF so just return the
2801  // widened first operand.
2802  return GetWidenedVector(N->getOperand(0));
2803 
2804  if (NumOperands == 2) {
2805  // Replace concat of two operands with a shuffle.
2806  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
2807  for (unsigned i = 0; i < NumInElts; ++i) {
2808  MaskOps[i] = i;
2809  MaskOps[i + NumInElts] = i + WidenNumElts;
2810  }
2811  return DAG.getVectorShuffle(WidenVT, dl,
2812  GetWidenedVector(N->getOperand(0)),
2813  GetWidenedVector(N->getOperand(1)),
2814  MaskOps);
2815  }
2816  }
2817  }
2818 
2819  // Fall back to use extracts and build vector.
2820  EVT EltVT = WidenVT.getVectorElementType();
2821  SmallVector<SDValue, 16> Ops(WidenNumElts);
2822  unsigned Idx = 0;
2823  for (unsigned i=0; i < NumOperands; ++i) {
2824  SDValue InOp = N->getOperand(i);
2825  if (InputWidened)
2826  InOp = GetWidenedVector(InOp);
2827  for (unsigned j=0; j < NumInElts; ++j)
2828  Ops[Idx++] = DAG.getNode(
2829  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2830  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2831  }
2832  SDValue UndefVal = DAG.getUNDEF(EltVT);
2833  for (; Idx < WidenNumElts; ++Idx)
2834  Ops[Idx] = UndefVal;
2835  return DAG.getBuildVector(WidenVT, dl, Ops);
2836 }
2837 
2838 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
2839  EVT VT = N->getValueType(0);
2840  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2841  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2842  SDValue InOp = N->getOperand(0);
2843  SDValue Idx = N->getOperand(1);
2844  SDLoc dl(N);
2845 
2846  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2847  InOp = GetWidenedVector(InOp);
2848 
2849  EVT InVT = InOp.getValueType();
2850 
2851  // Check if we can just return the input vector after widening.
2852  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2853  if (IdxVal == 0 && InVT == WidenVT)
2854  return InOp;
2855 
2856  // Check if we can extract from the vector.
2857  unsigned InNumElts = InVT.getVectorNumElements();
2858  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2859  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
2860 
2861  // We could try widening the input to the right length but for now, extract
2862  // the original elements, fill the rest with undefs and build a vector.
2863  SmallVector<SDValue, 16> Ops(WidenNumElts);
2864  EVT EltVT = VT.getVectorElementType();
2865  unsigned NumElts = VT.getVectorNumElements();
2866  unsigned i;
2867  for (i=0; i < NumElts; ++i)
2868  Ops[i] =
2869  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2870  DAG.getConstant(IdxVal + i, dl,
2871  TLI.getVectorIdxTy(DAG.getDataLayout())));
2872 
2873  SDValue UndefVal = DAG.getUNDEF(EltVT);
2874  for (; i < WidenNumElts; ++i)
2875  Ops[i] = UndefVal;
2876  return DAG.getBuildVector(WidenVT, dl, Ops);
2877 }
2878 
2879 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
2880  SDValue InOp = GetWidenedVector(N->getOperand(0));
2881  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
2882  InOp.getValueType(), InOp,
2883  N->getOperand(1), N->getOperand(2));
2884 }
2885 
2886 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
2887  LoadSDNode *LD = cast<LoadSDNode>(N);
2888  ISD::LoadExtType ExtType = LD->getExtensionType();
2889 
2890  SDValue Result;
2891  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
2892  if (ExtType != ISD::NON_EXTLOAD)
2893  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2894  else
2895  Result = GenWidenVectorLoads(LdChain, LD);
2896 
2897  // If we generate a single load, we can use that for the chain. Otherwise,
2898  // build a factor node to remember the multiple loads are independent and
2899  // chain to that.
2900  SDValue NewChain;
2901  if (LdChain.size() == 1)
2902  NewChain = LdChain[0];
2903  else
2904  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
2905 
2906  // Modified the chain - switch anything that used the old chain to use
2907  // the new one.
2908  ReplaceValueWith(SDValue(N, 1), NewChain);
2909 
2910  return Result;
2911 }
2912 
2913 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
2914 
2915  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
2916  SDValue Mask = N->getMask();
2917  EVT MaskVT = Mask.getValueType();
2918  SDValue Src0 = GetWidenedVector(N->getSrc0());
2919  ISD::LoadExtType ExtType = N->getExtensionType();
2920  SDLoc dl(N);
2921 
2922  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
2923  Mask = GetWidenedVector(Mask);
2924  else {
2925  EVT BoolVT = getSetCCResultType(WidenVT);
2926 
2927  // We can't use ModifyToType() because we should fill the mask with
2928  // zeroes
2929  unsigned WidenNumElts = BoolVT.getVectorNumElements();
2930  unsigned MaskNumElts = MaskVT.getVectorNumElements();
2931 
2932  unsigned NumConcat = WidenNumElts / MaskNumElts;
2933  SmallVector<SDValue, 16> Ops(NumConcat);
2934  SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
2935  Ops[0] = Mask;
2936  for (unsigned i = 1; i != NumConcat; ++i)
2937  Ops[i] = ZeroVal;
2938 
2939  Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
2940  }
2941 
2942  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
2943  Mask, Src0, N->getMemoryVT(),
2944  N->getMemOperand(), ExtType,
2945  N->isExpandingLoad());
2946  // Legalize the chain result - switch anything that used the old chain to
2947  // use the new one.
2948  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2949  return Res;
2950 }
2951 
2952 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
2953 
2954  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2955  SDValue Mask = N->getMask();
2956  SDValue Src0 = GetWidenedVector(N->getValue());
2957  unsigned NumElts = WideVT.getVectorNumElements();
2958  SDLoc dl(N);
2959 
2960  // The mask should be widened as well
2961  Mask = WidenTargetBoolean(Mask, WideVT, true);
2962 
2963  // Widen the Index operand
2964  SDValue Index = N->getIndex();
2965  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
2966  Index.getValueType().getScalarType(),
2967  NumElts);
2968  Index = ModifyToType(Index, WideIndexVT);
2969  SDValue Ops[] = { N->getChain(), Src0, Mask, N->getBasePtr(), Index };
2970  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
2971  N->getMemoryVT(), dl, Ops,
2972  N->getMemOperand());
2973 
2974  // Legalize the chain result - switch anything that used the old chain to
2975  // use the new one.
2976  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2977  return Res;
2978 }
2979 
2980 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
2981  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2982  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
2983  WidenVT, N->getOperand(0));
2984 }
2985 
2986 // Return true if this is a node that could have two SETCCs as operands.
2987 static inline bool isLogicalMaskOp(unsigned Opcode) {
2988  switch (Opcode) {
2989  case ISD::AND:
2990  case ISD::OR:
2991  case ISD::XOR:
2992  return true;
2993  }
2994  return false;
2995 }
2996 
2997 // This is used just for the assert in convertMask(). Check that this either
2998 // a SETCC or a previously handled SETCC by convertMask().
2999 #ifndef NDEBUG
3000 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3001  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3002  N = N.getOperand(0);
3003  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3004  for (unsigned i = 1; i < N->getNumOperands(); ++i)
3005  if (!N->getOperand(i)->isUndef())
3006  return false;
3007  N = N.getOperand(0);
3008  }
3009 
3010  if (N.getOpcode() == ISD::TRUNCATE)
3011  N = N.getOperand(0);
3012  else if (N.getOpcode() == ISD::SIGN_EXTEND)
3013  N = N.getOperand(0);
3014 
3015  if (isLogicalMaskOp(N.getOpcode()))
3016  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3018 
3019  return (N.getOpcode() == ISD::SETCC ||
3021 }
3022 #endif
3023 
3024 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3025 // to ToMaskVT if needed with vector extension or truncation.
3026 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3027  EVT ToMaskVT) {
3028  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3029  // FIXME: This code seems to be too restrictive, we might consider
3030  // generalizing it or dropping it.
3031  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3032 
3033  // Make a new Mask node, with a legal result VT.
3035  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3036  Ops.push_back(InMask->getOperand(i));
3037  SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3038 
3039  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3040  // extend or truncate is needed.
3041  LLVMContext &Ctx = *DAG.getContext();
3042  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3043  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3044  if (MaskScalarBits < ToMaskScalBits) {
3045  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3046  MaskVT.getVectorNumElements());
3047  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3048  } else if (MaskScalarBits > ToMaskScalBits) {
3049  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3050  MaskVT.getVectorNumElements());
3051  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3052  }
3053 
3055  ToMaskVT.getScalarSizeInBits() &&
3056  "Mask should have the right element size by now.");
3057 
3058  // Adjust Mask to the right number of elements.
3059  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3060  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3061  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3062  SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3063  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3064  ZeroIdx);
3065  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3066  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3067  EVT SubVT = Mask->getValueType(0);
3068  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3069  SubOps[0] = Mask;
3070  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3071  }
3072 
3073  assert((Mask->getValueType(0) == ToMaskVT) &&
3074  "A mask of ToMaskVT should have been produced by now.");
3075 
3076  return Mask;
3077 }
3078 
3079 // Get the target mask VT, and widen if needed.
3080 EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
3081  assert(SetCC->getOpcode() == ISD::SETCC);
3082  LLVMContext &Ctx = *DAG.getContext();
3083  EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
3084  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3085  MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
3086  return MaskVT;
3087 }
3088 
3089 // This method tries to handle VSELECT and its mask by legalizing operands
3090 // (which may require widening) and if needed adjusting the mask vector type
3091 // to match that of the VSELECT. Without it, many cases end up with
3092 // scalarization of the SETCC, with many unnecessary instructions.
3093 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3094  LLVMContext &Ctx = *DAG.getContext();
3095  SDValue Cond = N->getOperand(0);
3096 
3097  if (N->getOpcode() != ISD::VSELECT)
3098  return SDValue();
3099 
3100  if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3101  return SDValue();
3102 
3103  // If this is a splitted VSELECT that was previously already handled, do
3104  // nothing.
3105  EVT CondVT = Cond->getValueType(0);
3106  if (CondVT.getScalarSizeInBits() != 1)
3107  return SDValue();
3108 
3109  EVT VSelVT = N->getValueType(0);
3110  // Only handle vector types which are a power of 2.
3111  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3112  return SDValue();
3113 
3114  // Don't touch if this will be scalarized.
3115  EVT FinalVT = VSelVT;
3116  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3117  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3118 
3119  if (FinalVT.getVectorNumElements() == 1)
3120  return SDValue();
3121 
3122  // If there is support for an i1 vector mask, don't touch.
3123  if (Cond.getOpcode() == ISD::SETCC) {
3124  EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3125  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3126  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3127  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3128  if (SetCCResVT.getScalarSizeInBits() == 1)
3129  return SDValue();
3130  } else if (CondVT.getScalarType() == MVT::i1) {
3131  // If there is support for an i1 vector mask (or only scalar i1 conditions),
3132  // don't touch.
3133  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3134  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3135 
3136  if (CondVT.getScalarType() == MVT::i1)
3137  return SDValue();
3138  }
3139 
3140  // Get the VT and operands for VSELECT, and widen if needed.
3141  SDValue VSelOp1 = N->getOperand(1);
3142  SDValue VSelOp2 = N->getOperand(2);
3143  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3144  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3145  VSelOp1 = GetWidenedVector(VSelOp1);
3146  VSelOp2 = GetWidenedVector(VSelOp2);
3147  }
3148 
3149  // The mask of the VSELECT should have integer elements.
3150  EVT ToMaskVT = VSelVT;
3151  if (!ToMaskVT.getScalarType().isInteger())
3152  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3153 
3154  SDValue Mask;
3155  if (Cond->getOpcode() == ISD::SETCC) {
3156  EVT MaskVT = getSETCCWidenedResultTy(Cond);
3157  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3158  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3159  Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3160  Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3161  // Cond is (AND/OR/XOR (SETCC, SETCC))
3162  SDValue SETCC0 = Cond->getOperand(0);
3163  SDValue SETCC1 = Cond->getOperand(1);
3164  EVT VT0 = getSETCCWidenedResultTy(SETCC0);
3165  EVT VT1 = getSETCCWidenedResultTy(SETCC1);
3166  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3167  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3168  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3169  EVT MaskVT;
3170  // If the two SETCCs have different VTs, either extend/truncate one of
3171  // them to the other "towards" ToMaskVT, or truncate one and extend the
3172  // other to ToMaskVT.
3173  if (ScalarBits0 != ScalarBits1) {
3174  EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3175  EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3176  if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3177  MaskVT = WideVT;
3178  else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3179  MaskVT = NarrowVT;
3180  else
3181  MaskVT = ToMaskVT;
3182  } else
3183  // If the two SETCCs have the same VT, don't change it.
3184  MaskVT = VT0;
3185 
3186  // Make new SETCCs and logical nodes.
3187  SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3188  SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3189  Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3190 
3191  // Convert the logical op for VSELECT if needed.
3192  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3193  } else
3194  return SDValue();
3195 
3196  return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3197 }
3198 
3199 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3200  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3201  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3202 
3203  SDValue Cond1 = N->getOperand(0);
3204  EVT CondVT = Cond1.getValueType();
3205  if (CondVT.isVector()) {
3206  if (SDValue Res = WidenVSELECTAndMask(N))
3207  return Res;
3208 
3209  EVT CondEltVT = CondVT.getVectorElementType();
3210  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3211  CondEltVT, WidenNumElts);
3212  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3213  Cond1 = GetWidenedVector(Cond1);
3214 
3215  // If we have to split the condition there is no point in widening the
3216  // select. This would result in an cycle of widening the select ->
3217  // widening the condition operand -> splitting the condition operand ->
3218  // splitting the select -> widening the select. Instead split this select
3219  // further and widen the resulting type.
3220  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3221  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3222  SDValue Res = ModifyToType(SplitSelect, WidenVT);
3223  return Res;
3224  }
3225 
3226  if (Cond1.getValueType() != CondWidenVT)
3227  Cond1 = ModifyToType(Cond1, CondWidenVT);
3228  }
3229 
3230  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3231  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3232  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3233  return DAG.getNode(N->getOpcode(), SDLoc(N),
3234  WidenVT, Cond1, InOp1, InOp2);
3235 }
3236 
3237 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3238  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3239  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3240  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3241  InOp1.getValueType(), N->getOperand(0),
3242  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3243 }
3244 
3245 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
3246  assert(N->getValueType(0).isVector() ==
3247  N->getOperand(0).getValueType().isVector() &&
3248  "Scalar/Vector type mismatch");
3249  if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N);
3250 
3251  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3252  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3253  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3254  return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT,
3255  InOp1, InOp2, N->getOperand(2));
3256 }
3257 
3258 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3259  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3260  return DAG.getUNDEF(WidenVT);
3261 }
3262 
3263 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3264  EVT VT = N->getValueType(0);
3265  SDLoc dl(N);
3266 
3267  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3268  unsigned NumElts = VT.getVectorNumElements();
3269  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3270 
3271  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3272  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3273 
3274  // Adjust mask based on new input vector length.
3275  SmallVector<int, 16> NewMask;
3276  for (unsigned i = 0; i != NumElts; ++i) {
3277  int Idx = N->getMaskElt(i);
3278  if (Idx < (int)NumElts)
3279  NewMask.push_back(Idx);
3280  else
3281  NewMask.push_back(Idx - NumElts + WidenNumElts);
3282  }
3283  for (unsigned i = NumElts; i != WidenNumElts; ++i)
3284  NewMask.push_back(-1);
3285  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
3286 }
3287 
3288 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
3289  assert(N->getValueType(0).isVector() &&
3290  N->getOperand(0).getValueType().isVector() &&
3291  "Operands must be vectors");
3292  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3293  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3294 
3295  SDValue InOp1 = N->getOperand(0);
3296  EVT InVT = InOp1.getValueType();
3297  assert(InVT.isVector() && "can not widen non-vector type");
3298  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
3299  InVT.getVectorElementType(), WidenNumElts);
3300 
3301  // The input and output types often differ here, and it could be that while
3302  // we'd prefer to widen the result type, the input operands have been split.
3303  // In this case, we also need to split the result of this node as well.
3304  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
3305  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
3306  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
3307  return Res;
3308  }
3309 
3310  InOp1 = GetWidenedVector(InOp1);
3311  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3312 
3313  // Assume that the input and output will be widen appropriately. If not,
3314  // we will have to unroll it at some point.
3315  assert(InOp1.getValueType() == WidenInVT &&
3316  InOp2.getValueType() == WidenInVT &&
3317  "Input not widened to expected type!");
3318  (void)WidenInVT;
3319  return DAG.getNode(ISD::SETCC, SDLoc(N),
3320  WidenVT, InOp1, InOp2, N->getOperand(2));
3321 }
3322 
3323 
3324 //===----------------------------------------------------------------------===//
3325 // Widen Vector Operand
3326 //===----------------------------------------------------------------------===//
3327 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
3328  DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
3329  N->dump(&DAG);
3330  dbgs() << "\n");
3331  SDValue Res = SDValue();
3332 
3333  // See if the target wants to custom widen this node.
3334  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3335  return false;
3336 
3337  switch (N->getOpcode()) {
3338  default:
3339 #ifndef NDEBUG
3340  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
3341  N->dump(&DAG);
3342  dbgs() << "\n";
3343 #endif
3344  llvm_unreachable("Do not know how to widen this operator's operand!");
3345 
3346  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
3347  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
3348  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
3349  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
3350  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
3351  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
3352  case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
3353  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
3354  case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
3355 
3356  case ISD::ANY_EXTEND:
3357  case ISD::SIGN_EXTEND:
3358  case ISD::ZERO_EXTEND:
3359  Res = WidenVecOp_EXTEND(N);
3360  break;
3361 
3362  case ISD::FP_EXTEND:
3363  case ISD::FP_TO_SINT:
3364  case ISD::FP_TO_UINT:
3365  case ISD::SINT_TO_FP:
3366  case ISD::UINT_TO_FP:
3367  case ISD::TRUNCATE:
3368  Res = WidenVecOp_Convert(N);
3369  break;
3370  }
3371 
3372  // If Res is null, the sub-method took care of registering the result.
3373  if (!Res.getNode()) return false;
3374 
3375  // If the result is N, the sub-method updated N in place. Tell the legalizer
3376  // core about this.
3377  if (Res.getNode() == N)
3378  return true;
3379 
3380 
3381  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3382  "Invalid operand expansion");
3383 
3384  ReplaceValueWith(SDValue(N, 0), Res);
3385  return false;
3386 }
3387 
3388 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
3389  SDLoc DL(N);
3390  EVT VT = N->getValueType(0);
3391 
3392  SDValue InOp = N->getOperand(0);
3393  // If some legalization strategy other than widening is used on the operand,
3394  // we can't safely assume that just extending the low lanes is the correct
3395  // transformation.
3396  if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector)
3397  return WidenVecOp_Convert(N);
3398  InOp = GetWidenedVector(InOp);
3401  "Input wasn't widened!");
3402 
3403  // We may need to further widen the operand until it has the same total
3404  // vector size as the result.
3405  EVT InVT = InOp.getValueType();
3406  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
3407  EVT InEltVT = InVT.getVectorElementType();
3408  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
3409  EVT FixedVT = (MVT::SimpleValueType)i;
3410  EVT FixedEltVT = FixedVT.getVectorElementType();
3411  if (TLI.isTypeLegal(FixedVT) &&
3412  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
3413  FixedEltVT == InEltVT) {
3414  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
3415  "Not enough elements in the fixed type for the operand!");
3416  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
3417  "We can't have the same type as we started with!");
3418  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
3419  InOp = DAG.getNode(
3420  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
3421  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3422  else
3423  InOp = DAG.getNode(
3424  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
3425  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3426  break;
3427  }
3428  }
3429  InVT = InOp.getValueType();
3430  if (InVT.getSizeInBits() != VT.getSizeInBits())
3431  // We couldn't find a legal vector type that was a widening of the input
3432  // and could be extended in-register to the result type, so we have to
3433  // scalarize.
3434  return WidenVecOp_Convert(N);
3435  }
3436 
3437  // Use special DAG nodes to represent the operation of extending the
3438  // low lanes.
3439  switch (N->getOpcode()) {
3440  default:
3441  llvm_unreachable("Extend legalization on on extend operation!");
3442  case ISD::ANY_EXTEND:
3443  return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
3444  case ISD::SIGN_EXTEND:
3445  return DAG.getSignExtendVectorInReg(InOp, DL, VT);
3446  case ISD::ZERO_EXTEND:
3447  return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
3448  }
3449 }
3450 
3451 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
3452  // The result (and first input) is legal, but the second input is illegal.
3453  // We can't do much to fix that, so just unroll and let the extracts off of
3454  // the second input be widened as needed later.
3455  return DAG.UnrollVectorOp(N);
3456 }
3457 
3458 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
3459  // Since the result is legal and the input is illegal, it is unlikely that we
3460  // can fix the input to a legal type so unroll the convert into some scalar
3461  // code and create a nasty build vector.
3462  EVT VT = N->getValueType(0);
3463  EVT EltVT = VT.getVectorElementType();
3464  SDLoc dl(N);
3465  unsigned NumElts = VT.getVectorNumElements();
3466  SDValue InOp = N->getOperand(0);
3467  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3468  InOp = GetWidenedVector(InOp);
3469  EVT InVT = InOp.getValueType();
3470  EVT InEltVT = InVT.getVectorElementType();
3471 
3472  unsigned Opcode = N->getOpcode();
3473  SmallVector<SDValue, 16> Ops(NumElts);
3474  for (unsigned i=0; i < NumElts; ++i)
3475  Ops[i] = DAG.getNode(
3476  Opcode, dl, EltVT,
3477  DAG.getNode(
3478  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
3479  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3480 
3481  return DAG.getBuildVector(VT, dl, Ops);
3482 }
3483 
3484 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
3485  EVT VT = N->getValueType(0);
3486  SDValue InOp = GetWidenedVector(N->getOperand(0));
3487  EVT InWidenVT = InOp.getValueType();
3488  SDLoc dl(N);
3489 
3490  // Check if we can convert between two legal vector types and extract.
3491  unsigned InWidenSize = InWidenVT.getSizeInBits();
3492  unsigned Size = VT.getSizeInBits();
3493  // x86mmx is not an acceptable vector element type, so don't try.
3494  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
3495  unsigned NewNumElts = InWidenSize / Size;
3496  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
3497  if (TLI.isTypeLegal(NewVT)) {
3498  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
3499  return DAG.getNode(
3500  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
3501  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3502  }
3503  }
3504 
3505  return CreateStackStoreLoad(InOp, VT);
3506 }
3507 
3508 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
3509  // If the input vector is not legal, it is likely that we will not find a
3510  // legal vector of the same size. Replace the concatenate vector with a
3511  // nasty build vector.
3512  EVT VT = N->getValueType(0);
3513  EVT EltVT = VT.getVectorElementType();
3514  SDLoc dl(N);
3515  unsigned NumElts = VT.getVectorNumElements();
3516  SmallVector<SDValue, 16> Ops(NumElts);
3517 
3518  EVT InVT = N->getOperand(0).getValueType();
3519  unsigned NumInElts = InVT.getVectorNumElements();
3520 
3521  unsigned Idx = 0;
3522  unsigned NumOperands = N->getNumOperands();
3523  for (unsigned i=0; i < NumOperands; ++i) {
3524  SDValue InOp = N->getOperand(i);
3525  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3526  InOp = GetWidenedVector(InOp);
3527  for (unsigned j=0; j < NumInElts; ++j)
3528  Ops[Idx++] = DAG.getNode(
3529  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3530  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3531  }
3532  return DAG.getBuildVector(VT, dl, Ops);
3533 }
3534 
3535 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3536  SDValue InOp = GetWidenedVector(N->getOperand(0));
3537  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
3538  N->getValueType(0), InOp, N->getOperand(1));
3539 }
3540 
3541 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3542  SDValue InOp = GetWidenedVector(N->getOperand(0));
3543  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3544  N->getValueType(0), InOp, N->getOperand(1));
3545 }
3546 
3547 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
3548  // We have to widen the value, but we want only to store the original
3549  // vector type.
3550  StoreSDNode *ST = cast<StoreSDNode>(N);
3551 
3552  SmallVector<SDValue, 16> StChain;
3553  if (ST->isTruncatingStore())
3554  GenWidenVectorTruncStores(StChain, ST);
3555  else
3556  GenWidenVectorStores(StChain, ST);
3557 
3558  if (StChain.size() == 1)
3559  return StChain[0];
3560  else
3561  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
3562 }
3563 
3564 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
3565  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
3566  SDValue Mask = MST->getMask();
3567  EVT MaskVT = Mask.getValueType();
3568  SDValue StVal = MST->getValue();
3569  // Widen the value
3570  SDValue WideVal = GetWidenedVector(StVal);
3571  SDLoc dl(N);
3572 
3573  if (OpNo == 2 || getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3574  Mask = GetWidenedVector(Mask);
3575  else {
3576  // The mask should be widened as well.
3577  EVT BoolVT = getSetCCResultType(WideVal.getValueType());
3578  // We can't use ModifyToType() because we should fill the mask with
3579  // zeroes.
3580  unsigned WidenNumElts = BoolVT.getVectorNumElements();
3581  unsigned MaskNumElts = MaskVT.getVectorNumElements();
3582 
3583  unsigned NumConcat = WidenNumElts / MaskNumElts;
3584  SmallVector<SDValue, 16> Ops(NumConcat);
3585  SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
3586  Ops[0] = Mask;
3587  for (unsigned i = 1; i != NumConcat; ++i)
3588  Ops[i] = ZeroVal;
3589 
3590  Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
3591  }
3592  assert(Mask.getValueType().getVectorNumElements() ==
3593  WideVal.getValueType().getVectorNumElements() &&
3594  "Mask and data vectors should have the same number of elements");
3595  return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
3596  Mask, MST->getMemoryVT(), MST->getMemOperand(),
3597  false, MST->isCompressingStore());
3598 }
3599 
3600 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
3601  assert(OpNo == 1 && "Can widen only data operand of mscatter");
3602  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
3603  SDValue DataOp = MSC->getValue();
3604  SDValue Mask = MSC->getMask();
3605 
3606  // Widen the value.
3607  SDValue WideVal = GetWidenedVector(DataOp);
3608  EVT WideVT = WideVal.getValueType();
3609  unsigned NumElts = WideVal.getValueType().getVectorNumElements();
3610  SDLoc dl(N);
3611 
3612  // The mask should be widened as well.
3613  Mask = WidenTargetBoolean(Mask, WideVT, true);
3614 
3615  // Widen index.
3616  SDValue Index = MSC->getIndex();
3617  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3618  Index.getValueType().getScalarType(),
3619  NumElts);
3620  Index = ModifyToType(Index, WideIndexVT);
3621 
3622  SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index};
3623  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
3624  MSC->getMemoryVT(), dl, Ops,
3625  MSC->getMemOperand());
3626 }
3627 
3628 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
3629  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
3630  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3631  SDLoc dl(N);
3632 
3633  // WARNING: In this code we widen the compare instruction with garbage.
3634  // This garbage may contain denormal floats which may be slow. Is this a real
3635  // concern ? Should we zero the unused lanes if this is a float compare ?
3636 
3637  // Get a new SETCC node to compare the newly widened operands.
3638  // Only some of the compared elements are legal.
3639  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3640  InOp0.getValueType());
3641  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
3642  SVT, InOp0, InOp1, N->getOperand(2));
3643 
3644  // Extract the needed results from the result vector.
3645  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
3646  SVT.getVectorElementType(),
3648  SDValue CC = DAG.getNode(
3649  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
3650  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3651 
3652  return PromoteTargetBoolean(CC, N->getValueType(0));
3653 }
3654 
3655 
3656 //===----------------------------------------------------------------------===//
3657 // Vector Widening Utilities
3658 //===----------------------------------------------------------------------===//
3659 
3660 // Utility function to find the type to chop up a widen vector for load/store
3661 // TLI: Target lowering used to determine legal types.
3662 // Width: Width left need to load/store.
3663 // WidenVT: The widen vector type to load to/store from
3664 // Align: If 0, don't allow use of a wider type
3665 // WidenEx: If Align is not 0, the amount additional we can load/store from.
3666 
3667 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
3668  unsigned Width, EVT WidenVT,
3669  unsigned Align = 0, unsigned WidenEx = 0) {
3670  EVT WidenEltVT = WidenVT.getVectorElementType();
3671  unsigned WidenWidth = WidenVT.getSizeInBits();
3672  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
3673  unsigned AlignInBits = Align*8;
3674 
3675  // If we have one element to load/store, return it.
3676  EVT RetVT = WidenEltVT;
3677  if (Width == WidenEltWidth)
3678  return RetVT;
3679 
3680  // See if there is larger legal integer than the element type to load/store.
3681  unsigned VT;
3682  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
3683  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
3684  EVT MemVT((MVT::SimpleValueType) VT);
3685  unsigned MemVTWidth = MemVT.getSizeInBits();
3686  if (MemVT.getSizeInBits() <= WidenEltWidth)
3687  break;
3688  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
3689  if ((Action == TargetLowering::TypeLegal ||
3691  (WidenWidth % MemVTWidth) == 0 &&
3692  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3693  (MemVTWidth <= Width ||
3694  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3695  RetVT = MemVT;
3696  break;
3697  }
3698  }
3699 
3700  // See if there is a larger vector type to load/store that has the same vector
3701  // element type and is evenly divisible with the WidenVT.
3702  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
3703  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
3704  EVT MemVT = (MVT::SimpleValueType) VT;
3705  unsigned MemVTWidth = MemVT.getSizeInBits();
3706  if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
3707  (WidenWidth % MemVTWidth) == 0 &&
3708  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3709  (MemVTWidth <= Width ||
3710  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3711  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
3712  return MemVT;
3713  }
3714  }
3715 
3716  return RetVT;
3717 }
3718 
3719 // Builds a vector type from scalar loads
3720 // VecTy: Resulting Vector type
3721 // LDOps: Load operators to build a vector type
3722 // [Start,End) the list of loads to use.
3724  SmallVectorImpl<SDValue> &LdOps,
3725  unsigned Start, unsigned End) {
3726  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3727  SDLoc dl(LdOps[Start]);
3728  EVT LdTy = LdOps[Start].getValueType();
3729  unsigned Width = VecTy.getSizeInBits();
3730  unsigned NumElts = Width / LdTy.getSizeInBits();
3731  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
3732 
3733  unsigned Idx = 1;
3734  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
3735 
3736  for (unsigned i = Start + 1; i != End; ++i) {
3737  EVT NewLdTy = LdOps[i].getValueType();
3738  if (NewLdTy != LdTy) {
3739  NumElts = Width / NewLdTy.getSizeInBits();
3740  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
3741  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
3742  // Readjust position and vector position based on new load type.
3743  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
3744  LdTy = NewLdTy;
3745  }
3746  VecOp = DAG.getNode(
3747  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
3748  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3749  }
3750  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
3751 }
3752 
3753 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
3754  LoadSDNode *LD) {
3755  // The strategy assumes that we can efficiently load power-of-two widths.
3756  // The routine chops the vector into the largest vector loads with the same
3757  // element type or scalar loads and then recombines it to the widen vector
3758  // type.
3759  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3760  unsigned WidenWidth = WidenVT.getSizeInBits();
3761  EVT LdVT = LD->getMemoryVT();
3762  SDLoc dl(LD);
3763  assert(LdVT.isVector() && WidenVT.isVector());
3764  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
3765 
3766  // Load information
3767  SDValue Chain = LD->getChain();
3768  SDValue BasePtr = LD->getBasePtr();
3769  unsigned Align = LD->getAlignment();
3770  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3771  AAMDNodes AAInfo = LD->getAAInfo();
3772 
3773  int LdWidth = LdVT.getSizeInBits();
3774  int WidthDiff = WidenWidth - LdWidth;
3775  unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
3776 
3777  // Find the vector type that can load from.
3778  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3779  int NewVTWidth = NewVT.getSizeInBits();
3780  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
3781  Align, MMOFlags, AAInfo);
3782  LdChain.push_back(LdOp.getValue(1));
3783 
3784  // Check if we can load the element with one instruction.
3785  if (LdWidth <= NewVTWidth) {
3786  if (!NewVT.isVector()) {
3787  unsigned NumElts = WidenWidth / NewVTWidth;
3788  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
3789  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
3790  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
3791  }
3792  if (NewVT == WidenVT)
3793  return LdOp;
3794 
3795  assert(WidenWidth % NewVTWidth == 0);
3796  unsigned NumConcat = WidenWidth / NewVTWidth;
3797  SmallVector<SDValue, 16> ConcatOps(NumConcat);
3798  SDValue UndefVal = DAG.getUNDEF(NewVT);
3799  ConcatOps[0] = LdOp;
3800  for (unsigned i = 1; i != NumConcat; ++i)
3801  ConcatOps[i] = UndefVal;
3802  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
3803  }
3804 
3805  // Load vector by using multiple loads from largest vector to scalar.
3807  LdOps.push_back(LdOp);
3808 
3809  LdWidth -= NewVTWidth;
3810  unsigned Offset = 0;
3811 
3812  while (LdWidth > 0) {
3813  unsigned Increment = NewVTWidth / 8;
3814  Offset += Increment;
3815  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3816  DAG.getConstant(Increment, dl, BasePtr.getValueType()));
3817 
3818  SDValue L;
3819  if (LdWidth < NewVTWidth) {
3820  // The current type we are using is too large. Find a better size.
3821  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3822  NewVTWidth = NewVT.getSizeInBits();
3823  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3824  LD->getPointerInfo().getWithOffset(Offset),
3825  MinAlign(Align, Increment), MMOFlags, AAInfo);
3826  LdChain.push_back(L.getValue(1));
3827  if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
3828  // Later code assumes the vector loads produced will be mergeable, so we
3829  // must pad the final entry up to the previous width. Scalars are
3830  // combined separately.
3832  Loads.push_back(L);
3833  unsigned size = L->getValueSizeInBits(0);
3834  while (size < LdOp->getValueSizeInBits(0)) {
3835  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
3836  size += L->getValueSizeInBits(0);
3837  }
3838  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
3839  }
3840  } else {
3841  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3842  LD->getPointerInfo().getWithOffset(Offset),
3843  MinAlign(Align, Increment), MMOFlags, AAInfo);
3844  LdChain.push_back(L.getValue(1));
3845  }
3846 
3847  LdOps.push_back(L);
3848  LdOp = L;
3849 
3850  LdWidth -= NewVTWidth;
3851  }
3852 
3853  // Build the vector from the load operations.
3854  unsigned End = LdOps.size();
3855  if (!LdOps[0].getValueType().isVector())
3856  // All the loads are scalar loads.
3857  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
3858 
3859  // If the load contains vectors, build the vector using concat vector.
3860  // All of the vectors used to load are power-of-2, and the scalar loads can be
3861  // combined to make a power-of-2 vector.
3862  SmallVector<SDValue, 16> ConcatOps(End);
3863  int i = End - 1;
3864  int Idx = End;
3865  EVT LdTy = LdOps[i].getValueType();
3866  // First, combine the scalar loads to a vector.
3867  if (!LdTy.isVector()) {
3868  for (--i; i >= 0; --i) {
3869  LdTy = LdOps[i].getValueType();
3870  if (LdTy.isVector())
3871  break;
3872  }
3873  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
3874  }
3875  ConcatOps[--Idx] = LdOps[i];
3876  for (--i; i >= 0; --i) {
3877  EVT NewLdTy = LdOps[i].getValueType();
3878  if (NewLdTy != LdTy) {
3879  // Create a larger vector.
3880  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
3881  makeArrayRef(&ConcatOps[Idx], End - Idx));
3882  Idx = End - 1;
3883  LdTy = NewLdTy;
3884  }
3885  ConcatOps[--Idx] = LdOps[i];
3886  }
3887 
3888  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
3889  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3890  makeArrayRef(&ConcatOps[Idx], End - Idx));
3891 
3892  // We need to fill the rest with undefs to build the vector.
3893  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
3894  SmallVector<SDValue, 16> WidenOps(NumOps);
3895  SDValue UndefVal = DAG.getUNDEF(LdTy);
3896  {
3897  unsigned i = 0;
3898  for (; i != End-Idx; ++i)
3899  WidenOps[i] = ConcatOps[Idx+i];
3900  for (; i != NumOps; ++i)
3901  WidenOps[i] = UndefVal;
3902  }
3903  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
3904 }
3905 
3906 SDValue
3907 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
3908  LoadSDNode *LD,
3909  ISD::LoadExtType ExtType) {
3910  // For extension loads, it may not be more efficient to chop up the vector
3911  // and then extend it. Instead, we unroll the load and build a new vector.
3912  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3913  EVT LdVT = LD->getMemoryVT();
3914  SDLoc dl(LD);
3915  assert(LdVT.isVector() && WidenVT.isVector());
3916 
3917  // Load information
3918  SDValue Chain = LD->getChain();
3919  SDValue BasePtr = LD->getBasePtr();
3920  unsigned Align = LD->getAlignment();
3921  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3922  AAMDNodes AAInfo = LD->getAAInfo();
3923 
3924  EVT EltVT = WidenVT.getVectorElementType();
3925  EVT LdEltVT = LdVT.getVectorElementType();
3926  unsigned NumElts = LdVT.getVectorNumElements();
3927 
3928  // Load each element and widen.
3929  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3930  SmallVector<SDValue, 16> Ops(WidenNumElts);
3931  unsigned Increment = LdEltVT.getSizeInBits() / 8;
3932  Ops[0] =
3933  DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
3934  LdEltVT, Align, MMOFlags, AAInfo);
3935  LdChain.push_back(Ops[0].getValue(1));
3936  unsigned i = 0, Offset = Increment;
3937  for (i=1; i < NumElts; ++i, Offset += Increment) {
3938  SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
3939  BasePtr,
3940  DAG.getConstant(Offset, dl,
3941  BasePtr.getValueType()));
3942  Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
3943  LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
3944  Align, MMOFlags, AAInfo);
3945  LdChain.push_back(Ops[i].getValue(1));
3946  }
3947 
3948  // Fill the rest with undefs.
3949  SDValue UndefVal = DAG.getUNDEF(EltVT);
3950  for (; i != WidenNumElts; ++i)
3951  Ops[i] = UndefVal;
3952 
3953  return DAG.getBuildVector(WidenVT, dl, Ops);
3954 }
3955 
3956 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
3957  StoreSDNode *ST) {
3958  // The strategy assumes that we can efficiently store power-of-two widths.
3959  // The routine chops the vector into the largest vector stores with the same
3960  // element type or scalar stores.
3961  SDValue Chain = ST->getChain();
3962  SDValue BasePtr = ST->getBasePtr();
3963  unsigned Align = ST->getAlignment();
3964  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
3965  AAMDNodes AAInfo = ST->getAAInfo();
3966  SDValue ValOp = GetWidenedVector(ST->getValue());
3967  SDLoc dl(ST);
3968 
3969  EVT StVT = ST->getMemoryVT();
3970  unsigned StWidth = StVT.getSizeInBits();
3971  EVT ValVT = ValOp.getValueType();
3972  unsigned ValWidth = ValVT.getSizeInBits();
3973  EVT ValEltVT = ValVT.getVectorElementType();
3974  unsigned ValEltWidth = ValEltVT.getSizeInBits();
3975  assert(StVT.getVectorElementType() == ValEltVT);
3976 
3977  int Idx = 0; // current index to store
3978  unsigned Offset = 0; // offset from base to store
3979  while (StWidth != 0) {
3980  // Find the largest vector type we can store with.
3981  EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
3982  unsigned NewVTWidth = NewVT.getSizeInBits();
3983  unsigned Increment = NewVTWidth / 8;
3984  if (NewVT.isVector()) {
3985  unsigned NumVTElts = NewVT.getVectorNumElements();
3986  do {
3987  SDValue EOp = DAG.getNode(
3988  ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
3989  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3990  StChain.push_back(DAG.getStore(
3991  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
3992  MinAlign(Align, Offset), MMOFlags, AAInfo));
3993  StWidth -= NewVTWidth;
3994  Offset += Increment;
3995  Idx += NumVTElts;
3996  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3997  DAG.getConstant(Increment, dl,
3998  BasePtr.getValueType()));
3999  } while (StWidth != 0 && StWidth >= NewVTWidth);
4000  } else {
4001  // Cast the vector to the scalar type we can store.
4002  unsigned NumElts = ValWidth / NewVTWidth;
4003  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4004  SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4005  // Readjust index position based on new vector type.
4006  Idx = Idx * ValEltWidth / NewVTWidth;
4007  do {
4008  SDValue EOp = DAG.getNode(
4009  ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4010  DAG.getConstant(Idx++, dl,
4011  TLI.getVectorIdxTy(DAG.getDataLayout())));
4012  StChain.push_back(DAG.getStore(
4013  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4014  MinAlign(Align, Offset), MMOFlags, AAInfo));
4015  StWidth -= NewVTWidth;
4016  Offset += Increment;
4017  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
4018  DAG.getConstant(Increment, dl,
4019  BasePtr.getValueType()));
4020  } while (StWidth != 0 && StWidth >= NewVTWidth);
4021  // Restore index back to be relative to the original widen element type.
4022  Idx = Idx * NewVTWidth / ValEltWidth;
4023  }
4024  }
4025 }
4026 
4027 void
4028 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4029  StoreSDNode *ST) {
4030  // For extension loads, it may not be more efficient to truncate the vector
4031  // and then store it. Instead, we extract each element and then store it.
4032  SDValue Chain = ST->getChain();
4033  SDValue BasePtr = ST->getBasePtr();
4034  unsigned Align = ST->getAlignment();
4035  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4036  AAMDNodes AAInfo = ST->getAAInfo();
4037  SDValue ValOp = GetWidenedVector(ST->getValue());
4038  SDLoc dl(ST);
4039 
4040  EVT StVT = ST->getMemoryVT();
4041  EVT ValVT = ValOp.getValueType();
4042 
4043  // It must be true that the wide vector type is bigger than where we need to
4044  // store.
4045  assert(StVT.isVector() && ValOp.getValueType().isVector());
4046  assert(StVT.bitsLT(ValOp.getValueType()));
4047 
4048  // For truncating stores, we can not play the tricks of chopping legal vector
4049  // types and bitcast it to the right type. Instead, we unroll the store.
4050  EVT StEltVT = StVT.getVectorElementType();
4051  EVT ValEltVT = ValVT.getVectorElementType();
4052  unsigned Increment = ValEltVT.getSizeInBits() / 8;
4053  unsigned NumElts = StVT.getVectorNumElements();
4054  SDValue EOp = DAG.getNode(
4055  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4056  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4057  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4058  ST->getPointerInfo(), StEltVT, Align,
4059  MMOFlags, AAInfo));
4060  unsigned Offset = Increment;
4061  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4062  SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
4063  BasePtr,
4064  DAG.getConstant(Offset, dl,
4065  BasePtr.getValueType()));
4066  SDValue EOp = DAG.getNode(
4067  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4068  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4069  StChain.push_back(DAG.getTruncStore(
4070  Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4071  StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4072  }
4073 }
4074 
4075 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
4076 /// input vector must have the same element type as NVT.
4077 /// FillWithZeroes specifies that the vector should be widened with zeroes.
4078 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
4079  bool FillWithZeroes) {
4080  // Note that InOp might have been widened so it might already have
4081  // the right width or it might need be narrowed.
4082  EVT InVT = InOp.getValueType();
4084  "input and widen element type must match");
4085  SDLoc dl(InOp);
4086 
4087  // Check if InOp already has the right width.
4088  if (InVT == NVT)
4089  return InOp;
4090 
4091  unsigned InNumElts = InVT.getVectorNumElements();
4092  unsigned WidenNumElts = NVT.getVectorNumElements();
4093  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
4094  unsigned NumConcat = WidenNumElts / InNumElts;
4095  SmallVector<SDValue, 16> Ops(NumConcat);
4096  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
4097  DAG.getUNDEF(InVT);
4098  Ops[0] = InOp;
4099  for (unsigned i = 1; i != NumConcat; ++i)
4100  Ops[i] = FillVal;
4101 
4102  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
4103  }
4104 
4105  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
4106  return DAG.getNode(
4107  ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
4108  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4109 
4110  // Fall back to extract and build.
4111  SmallVector<SDValue, 16> Ops(WidenNumElts);
4112  EVT EltVT = NVT.getVectorElementType();
4113  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
4114  unsigned Idx;
4115  for (Idx = 0; Idx < MinNumElts; ++Idx)
4116  Ops[Idx] = DAG.getNode(
4117  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4118  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4119 
4120  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
4121  DAG.getUNDEF(EltVT);
4122  for ( ; Idx < WidenNumElts; ++Idx)
4123  Ops[Idx] = FillVal;
4124  return DAG.getBuildVector(NVT, dl, Ops);
4125 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:545
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:512
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:569
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isUndef() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:333
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:115
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:342
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:494
SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:351
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getBasePtr() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getChain() const
unsigned getAlignment() const
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:911
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:485
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:359
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:328
uint64_t High
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:404
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
const SDValue & getValue() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
A description of a memory reference used in the backend.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:379
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:205
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:446
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:388
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:385
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:455
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:398
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:830
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:558
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:472
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:498
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:302
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:602
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getBasePtr() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
These reductions are non-strict, and have a single vector operand.
Definition: ISDOpcodes.h:825
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:421
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:69
Simple binary floating point operators.
Definition: ISDOpcodes.h:259
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:891
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:307
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node&#39;s operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:692
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:426
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Idx) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition: ValueTypes.h:58
bool hasNoNaNs() const
static const unsigned End
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:385
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:530
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
const SDValue & getSrc0() const
bool isVolatile() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const SDValue & getMask() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
The memory access writes data.
SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:391
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:294
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:314
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:209
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:527
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:712
bool isVector(MCInstrInfo const &MCII, MCInst const &MCI)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:674
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:96
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:642
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:382
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
Definition: ValueTypes.h:342
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored...
Represents one node in the SelectionDAG.
static bool isSETCCorConvertedSETCC(SDValue N)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachinePointerInfo getWithOffset(int64_t O) const
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:923
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:389
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:445
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:448
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:287
int getMaskElt(unsigned Idx) const
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
Definition: ISDOpcodes.h:572
Flags
Flags values. These may be or&#39;d together.
amdgpu Simplify well known AMD library false Value Value * Arg
The memory access reads data.
These are IR-level optimization flags that may be propagated to SDNodes.
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:362
pointer data()
Return a pointer to the vector&#39;s buffer, even if empty().
Definition: SmallVector.h:143
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:463
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:581
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return &#39;Legal&#39;) or we ...
#define N
Flags getFlags() const
Return the raw flags of the source value,.
unsigned getOpcode() const
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:320
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:277
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:830
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
bool isUndef() const
Return true if the type of the node type undefined.
#define DEBUG(X)
Definition: Debug.h:118
ISD::LoadExtType getExtensionType() const
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:412
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
static bool isLogicalMaskOp(unsigned Opcode)
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:197
Conversion operators.
Definition: ISDOpcodes.h:442
BooleanContent
Enum that describes how the target represents true/false values.
const SDValue & getOperand(unsigned i) const
bool isBigEndian() const
Definition: DataLayout.h:217
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:451
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:561
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVMContext * getContext() const
Definition: SelectionDAG.h:393
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:350
const SDValue & getMask() const
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:355
This class is used to represent ISD::LOAD nodes.