LLVM  4.0.0
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16 // types.
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "legalize-types"
30 
31 //===----------------------------------------------------------------------===//
32 // Result Vector Scalarization: <1 x ty> -> ty.
33 //===----------------------------------------------------------------------===//
34 
35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36  DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
37  N->dump(&DAG);
38  dbgs() << "\n");
39  SDValue R = SDValue();
40 
41  switch (N->getOpcode()) {
42  default:
43 #ifndef NDEBUG
44  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45  N->dump(&DAG);
46  dbgs() << "\n";
47 #endif
48  report_fatal_error("Do not know how to scalarize the result of this "
49  "operator!\n");
50 
51  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
57  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
58  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
59  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
60  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
61  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
62  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
63  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
64  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
65  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
66  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
67  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
68  case ISD::ANY_EXTEND:
69  case ISD::BITREVERSE:
70  case ISD::BSWAP:
71  case ISD::CTLZ:
73  case ISD::CTPOP:
74  case ISD::CTTZ:
76  case ISD::FABS:
77  case ISD::FCEIL:
78  case ISD::FCOS:
79  case ISD::FEXP:
80  case ISD::FEXP2:
81  case ISD::FFLOOR:
82  case ISD::FLOG:
83  case ISD::FLOG10:
84  case ISD::FLOG2:
85  case ISD::FNEARBYINT:
86  case ISD::FNEG:
87  case ISD::FP_EXTEND:
88  case ISD::FP_TO_SINT:
89  case ISD::FP_TO_UINT:
90  case ISD::FRINT:
91  case ISD::FROUND:
92  case ISD::FSIN:
93  case ISD::FSQRT:
94  case ISD::FTRUNC:
95  case ISD::SIGN_EXTEND:
96  case ISD::SINT_TO_FP:
97  case ISD::TRUNCATE:
98  case ISD::UINT_TO_FP:
99  case ISD::ZERO_EXTEND:
100  R = ScalarizeVecRes_UnaryOp(N);
101  break;
102 
103  case ISD::ADD:
104  case ISD::AND:
105  case ISD::FADD:
106  case ISD::FCOPYSIGN:
107  case ISD::FDIV:
108  case ISD::FMUL:
109  case ISD::FMINNUM:
110  case ISD::FMAXNUM:
111  case ISD::FMINNAN:
112  case ISD::FMAXNAN:
113  case ISD::SMIN:
114  case ISD::SMAX:
115  case ISD::UMIN:
116  case ISD::UMAX:
117 
118  case ISD::FPOW:
119  case ISD::FREM:
120  case ISD::FSUB:
121  case ISD::MUL:
122  case ISD::OR:
123  case ISD::SDIV:
124  case ISD::SREM:
125  case ISD::SUB:
126  case ISD::UDIV:
127  case ISD::UREM:
128  case ISD::XOR:
129  case ISD::SHL:
130  case ISD::SRA:
131  case ISD::SRL:
132  R = ScalarizeVecRes_BinOp(N);
133  break;
134  case ISD::FMA:
135  R = ScalarizeVecRes_TernaryOp(N);
136  break;
137  }
138 
139  // If R is null, the sub-method took care of registering the result.
140  if (R.getNode())
141  SetScalarizedVector(SDValue(N, ResNo), R);
142 }
143 
144 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
145  SDValue LHS = GetScalarizedVector(N->getOperand(0));
146  SDValue RHS = GetScalarizedVector(N->getOperand(1));
147  return DAG.getNode(N->getOpcode(), SDLoc(N),
148  LHS.getValueType(), LHS, RHS, N->getFlags());
149 }
150 
151 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
152  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
153  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
154  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
155  return DAG.getNode(N->getOpcode(), SDLoc(N),
156  Op0.getValueType(), Op0, Op1, Op2);
157 }
158 
159 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
160  unsigned ResNo) {
161  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
162  return GetScalarizedVector(Op);
163 }
164 
165 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
166  EVT NewVT = N->getValueType(0).getVectorElementType();
167  return DAG.getNode(ISD::BITCAST, SDLoc(N),
168  NewVT, N->getOperand(0));
169 }
170 
171 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
172  EVT EltVT = N->getValueType(0).getVectorElementType();
173  SDValue InOp = N->getOperand(0);
174  // The BUILD_VECTOR operands may be of wider element types and
175  // we may need to truncate them back to the requested return type.
176  if (EltVT.isInteger())
177  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
178  return InOp;
179 }
180 
181 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
182  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
184  N->getOperand(0), N->getOperand(1));
185 }
186 
187 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
188  EVT NewVT = N->getValueType(0).getVectorElementType();
189  SDValue Op = GetScalarizedVector(N->getOperand(0));
190  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
191  NewVT, Op, N->getOperand(1));
192 }
193 
194 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
195  SDValue Op = GetScalarizedVector(N->getOperand(0));
196  return DAG.getNode(ISD::FPOWI, SDLoc(N),
197  Op.getValueType(), Op, N->getOperand(1));
198 }
199 
200 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
201  // The value to insert may have a wider type than the vector element type,
202  // so be sure to truncate it to the element type if necessary.
203  SDValue Op = N->getOperand(1);
204  EVT EltVT = N->getValueType(0).getVectorElementType();
205  if (Op.getValueType() != EltVT)
206  // FIXME: Can this happen for floating point types?
207  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
208  return Op;
209 }
210 
211 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
212  assert(N->isUnindexed() && "Indexed vector load?");
213 
214  SDValue Result = DAG.getLoad(
217  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
220  N->getAAInfo());
221 
222  // Legalize the chain result - switch anything that used the old chain to
223  // use the new one.
224  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
225  return Result;
226 }
227 
228 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
229  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
230  EVT DestVT = N->getValueType(0).getVectorElementType();
231  SDValue Op = N->getOperand(0);
232  EVT OpVT = Op.getValueType();
233  SDLoc DL(N);
234  // The result needs scalarizing, but it's not a given that the source does.
235  // This is a workaround for targets where it's impossible to scalarize the
236  // result of a conversion, because the source type is legal.
237  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
238  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
239  // legal and was not scalarized.
240  // See the similar logic in ScalarizeVecRes_VSETCC
241  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
242  Op = GetScalarizedVector(Op);
243  } else {
244  EVT VT = OpVT.getVectorElementType();
245  Op = DAG.getNode(
246  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
247  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
248  }
249  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
250 }
251 
252 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
253  EVT EltVT = N->getValueType(0).getVectorElementType();
254  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
255  SDValue LHS = GetScalarizedVector(N->getOperand(0));
256  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
257  LHS, DAG.getValueType(ExtVT));
258 }
259 
260 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
261  // If the operand is wider than the vector element type then it is implicitly
262  // truncated. Make that explicit here.
263  EVT EltVT = N->getValueType(0).getVectorElementType();
264  SDValue InOp = N->getOperand(0);
265  if (InOp.getValueType() != EltVT)
266  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
267  return InOp;
268 }
269 
270 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
271  SDValue Cond = GetScalarizedVector(N->getOperand(0));
272  SDValue LHS = GetScalarizedVector(N->getOperand(1));
273  TargetLowering::BooleanContent ScalarBool =
274  TLI.getBooleanContents(false, false);
275  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
276 
277  // If integer and float booleans have different contents then we can't
278  // reliably optimize in all cases. There is a full explanation for this in
279  // DAGCombiner::visitSELECT() where the same issue affects folding
280  // (select C, 0, 1) to (xor C, 1).
281  if (TLI.getBooleanContents(false, false) !=
282  TLI.getBooleanContents(false, true)) {
283  // At least try the common case where the boolean is generated by a
284  // comparison.
285  if (Cond->getOpcode() == ISD::SETCC) {
286  EVT OpVT = Cond->getOperand(0)->getValueType(0);
287  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
288  VecBool = TLI.getBooleanContents(OpVT);
289  } else
291  }
292 
293  if (ScalarBool != VecBool) {
294  EVT CondVT = Cond.getValueType();
295  switch (ScalarBool) {
297  break;
301  // Vector read from all ones, scalar expects a single 1 so mask.
302  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
303  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
304  break;
308  // Vector reads from a one, scalar from all ones so sign extend.
309  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
310  Cond, DAG.getValueType(MVT::i1));
311  break;
312  }
313  }
314 
315  return DAG.getSelect(SDLoc(N),
316  LHS.getValueType(), Cond, LHS,
317  GetScalarizedVector(N->getOperand(2)));
318 }
319 
320 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
321  SDValue LHS = GetScalarizedVector(N->getOperand(1));
322  return DAG.getSelect(SDLoc(N),
323  LHS.getValueType(), N->getOperand(0), LHS,
324  GetScalarizedVector(N->getOperand(2)));
325 }
326 
327 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
328  SDValue LHS = GetScalarizedVector(N->getOperand(2));
329  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
330  N->getOperand(0), N->getOperand(1),
331  LHS, GetScalarizedVector(N->getOperand(3)),
332  N->getOperand(4));
333 }
334 
335 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
336  assert(N->getValueType(0).isVector() ==
337  N->getOperand(0).getValueType().isVector() &&
338  "Scalar/Vector type mismatch");
339 
340  if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N);
341 
342  SDValue LHS = GetScalarizedVector(N->getOperand(0));
343  SDValue RHS = GetScalarizedVector(N->getOperand(1));
344  SDLoc DL(N);
345 
346  // Turn it into a scalar SETCC.
347  return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2));
348 }
349 
350 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
351  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
352 }
353 
354 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
355  // Figure out if the scalar is the LHS or RHS and return it.
356  SDValue Arg = N->getOperand(2).getOperand(0);
357  if (Arg.isUndef())
358  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
359  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
360  return GetScalarizedVector(N->getOperand(Op));
361 }
362 
363 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
364  assert(N->getValueType(0).isVector() &&
365  N->getOperand(0).getValueType().isVector() &&
366  "Operand types must be vectors");
367  SDValue LHS = N->getOperand(0);
368  SDValue RHS = N->getOperand(1);
369  EVT OpVT = LHS.getValueType();
370  EVT NVT = N->getValueType(0).getVectorElementType();
371  SDLoc DL(N);
372 
373  // The result needs scalarizing, but it's not a given that the source does.
374  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
375  LHS = GetScalarizedVector(LHS);
376  RHS = GetScalarizedVector(RHS);
377  } else {
378  EVT VT = OpVT.getVectorElementType();
379  LHS = DAG.getNode(
380  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
381  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
382  RHS = DAG.getNode(
383  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
384  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
385  }
386 
387  // Turn it into a scalar SETCC.
388  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
389  N->getOperand(2));
390  // Vectors may have a different boolean contents to scalars. Promote the
391  // value appropriately.
392  ISD::NodeType ExtendCode =
394  return DAG.getNode(ExtendCode, DL, NVT, Res);
395 }
396 
397 
398 //===----------------------------------------------------------------------===//
399 // Operand Vector Scalarization <1 x ty> -> ty.
400 //===----------------------------------------------------------------------===//
401 
402 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
403  DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
404  N->dump(&DAG);
405  dbgs() << "\n");
406  SDValue Res = SDValue();
407 
408  if (!Res.getNode()) {
409  switch (N->getOpcode()) {
410  default:
411 #ifndef NDEBUG
412  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
413  N->dump(&DAG);
414  dbgs() << "\n";
415 #endif
416  llvm_unreachable("Do not know how to scalarize this operator's operand!");
417  case ISD::BITCAST:
418  Res = ScalarizeVecOp_BITCAST(N);
419  break;
420  case ISD::ANY_EXTEND:
421  case ISD::ZERO_EXTEND:
422  case ISD::SIGN_EXTEND:
423  case ISD::TRUNCATE:
424  case ISD::FP_TO_SINT:
425  case ISD::FP_TO_UINT:
426  case ISD::SINT_TO_FP:
427  case ISD::UINT_TO_FP:
428  Res = ScalarizeVecOp_UnaryOp(N);
429  break;
430  case ISD::CONCAT_VECTORS:
431  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
432  break;
434  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
435  break;
436  case ISD::VSELECT:
437  Res = ScalarizeVecOp_VSELECT(N);
438  break;
439  case ISD::STORE:
440  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
441  break;
442  case ISD::FP_ROUND:
443  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
444  break;
445  }
446  }
447 
448  // If the result is null, the sub-method took care of registering results etc.
449  if (!Res.getNode()) return false;
450 
451  // If the result is N, the sub-method updated N in place. Tell the legalizer
452  // core about this.
453  if (Res.getNode() == N)
454  return true;
455 
456  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
457  "Invalid operand expansion");
458 
459  ReplaceValueWith(SDValue(N, 0), Res);
460  return false;
461 }
462 
463 /// If the value to convert is a vector that needs to be scalarized, it must be
464 /// <1 x ty>. Convert the element instead.
465 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
466  SDValue Elt = GetScalarizedVector(N->getOperand(0));
467  return DAG.getNode(ISD::BITCAST, SDLoc(N),
468  N->getValueType(0), Elt);
469 }
470 
471 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
472 /// Do the operation on the element instead.
473 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
475  "Unexpected vector type!");
476  SDValue Elt = GetScalarizedVector(N->getOperand(0));
477  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
478  N->getValueType(0).getScalarType(), Elt);
479  // Revectorize the result so the types line up with what the uses of this
480  // expression expect.
481  return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Op);
482 }
483 
484 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
485 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
487  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
488  Ops[i] = GetScalarizedVector(N->getOperand(i));
489  return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Ops);
490 }
491 
492 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
493 /// so just return the element, ignoring the index.
494 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
495  SDValue Res = GetScalarizedVector(N->getOperand(0));
496  if (Res.getValueType() != N->getValueType(0))
497  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0),
498  Res);
499  return Res;
500 }
501 
502 
503 /// If the input condition is a vector that needs to be scalarized, it must be
504 /// <1 x i1>, so just convert to a normal ISD::SELECT
505 /// (still with vector output type since that was acceptable if we got here).
506 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
507  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
508  EVT VT = N->getValueType(0);
509 
510  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
511  N->getOperand(2));
512 }
513 
514 /// If the value to store is a vector that needs to be scalarized, it must be
515 /// <1 x ty>. Just store the element.
516 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
517  assert(N->isUnindexed() && "Indexed store of one-element vector?");
518  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
519  SDLoc dl(N);
520 
521  if (N->isTruncatingStore())
522  return DAG.getTruncStore(
523  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
524  N->getBasePtr(), N->getPointerInfo(),
526  N->getMemOperand()->getFlags(), N->getAAInfo());
527 
528  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
529  N->getBasePtr(), N->getPointerInfo(),
531  N->getAAInfo());
532 }
533 
534 /// If the value to round is a vector that needs to be scalarized, it must be
535 /// <1 x ty>. Convert the element instead.
536 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
537  SDValue Elt = GetScalarizedVector(N->getOperand(0));
538  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
539  N->getValueType(0).getVectorElementType(), Elt,
540  N->getOperand(1));
541  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
542 }
543 
544 //===----------------------------------------------------------------------===//
545 // Result Vector Splitting
546 //===----------------------------------------------------------------------===//
547 
548 /// This method is called when the specified result of the specified node is
549 /// found to need vector splitting. At this point, the node may also have
550 /// invalid operands or may have other results that need legalization, we just
551 /// know that (at least) one result needs vector splitting.
552 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
553  DEBUG(dbgs() << "Split node result: ";
554  N->dump(&DAG);
555  dbgs() << "\n");
556  SDValue Lo, Hi;
557 
558  // See if the target wants to custom expand this node.
559  if (CustomLowerNode(N, N->getValueType(ResNo), true))
560  return;
561 
562  switch (N->getOpcode()) {
563  default:
564 #ifndef NDEBUG
565  dbgs() << "SplitVectorResult #" << ResNo << ": ";
566  N->dump(&DAG);
567  dbgs() << "\n";
568 #endif
569  report_fatal_error("Do not know how to split the result of this "
570  "operator!\n");
571 
572  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
573  case ISD::VSELECT:
574  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
575  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
576  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
577  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
578  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
579  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
580  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
581  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
582  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
583  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
584  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
585  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
586  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
587  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
588  case ISD::LOAD:
589  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
590  break;
591  case ISD::MLOAD:
592  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
593  break;
594  case ISD::MGATHER:
595  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
596  break;
597  case ISD::SETCC:
598  SplitVecRes_SETCC(N, Lo, Hi);
599  break;
600  case ISD::VECTOR_SHUFFLE:
601  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
602  break;
603 
607  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
608  break;
609 
610  case ISD::BITREVERSE:
611  case ISD::BSWAP:
612  case ISD::CTLZ:
613  case ISD::CTTZ:
616  case ISD::CTPOP:
617  case ISD::FABS:
618  case ISD::FCEIL:
619  case ISD::FCOS:
620  case ISD::FEXP:
621  case ISD::FEXP2:
622  case ISD::FFLOOR:
623  case ISD::FLOG:
624  case ISD::FLOG10:
625  case ISD::FLOG2:
626  case ISD::FNEARBYINT:
627  case ISD::FNEG:
628  case ISD::FP_EXTEND:
629  case ISD::FP_ROUND:
630  case ISD::FP_TO_SINT:
631  case ISD::FP_TO_UINT:
632  case ISD::FRINT:
633  case ISD::FROUND:
634  case ISD::FSIN:
635  case ISD::FSQRT:
636  case ISD::FTRUNC:
637  case ISD::SINT_TO_FP:
638  case ISD::TRUNCATE:
639  case ISD::UINT_TO_FP:
640  SplitVecRes_UnaryOp(N, Lo, Hi);
641  break;
642 
643  case ISD::ANY_EXTEND:
644  case ISD::SIGN_EXTEND:
645  case ISD::ZERO_EXTEND:
646  SplitVecRes_ExtendOp(N, Lo, Hi);
647  break;
648 
649  case ISD::ADD:
650  case ISD::SUB:
651  case ISD::MUL:
652  case ISD::MULHS:
653  case ISD::MULHU:
654  case ISD::FADD:
655  case ISD::FSUB:
656  case ISD::FMUL:
657  case ISD::FMINNUM:
658  case ISD::FMAXNUM:
659  case ISD::FMINNAN:
660  case ISD::FMAXNAN:
661  case ISD::SDIV:
662  case ISD::UDIV:
663  case ISD::FDIV:
664  case ISD::FPOW:
665  case ISD::AND:
666  case ISD::OR:
667  case ISD::XOR:
668  case ISD::SHL:
669  case ISD::SRA:
670  case ISD::SRL:
671  case ISD::UREM:
672  case ISD::SREM:
673  case ISD::FREM:
674  case ISD::SMIN:
675  case ISD::SMAX:
676  case ISD::UMIN:
677  case ISD::UMAX:
678  SplitVecRes_BinOp(N, Lo, Hi);
679  break;
680  case ISD::FMA:
681  SplitVecRes_TernaryOp(N, Lo, Hi);
682  break;
683  }
684 
685  // If Lo/Hi is null, the sub-method took care of registering results etc.
686  if (Lo.getNode())
687  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
688 }
689 
690 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
691  SDValue &Hi) {
692  SDValue LHSLo, LHSHi;
693  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
694  SDValue RHSLo, RHSHi;
695  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
696  SDLoc dl(N);
697 
698  const SDNodeFlags *Flags = N->getFlags();
699  unsigned Opcode = N->getOpcode();
700  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
701  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
702 }
703 
704 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
705  SDValue &Hi) {
706  SDValue Op0Lo, Op0Hi;
707  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
708  SDValue Op1Lo, Op1Hi;
709  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
710  SDValue Op2Lo, Op2Hi;
711  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
712  SDLoc dl(N);
713 
714  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
715  Op0Lo, Op1Lo, Op2Lo);
716  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
717  Op0Hi, Op1Hi, Op2Hi);
718 }
719 
720 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
721  SDValue &Hi) {
722  // We know the result is a vector. The input may be either a vector or a
723  // scalar value.
724  EVT LoVT, HiVT;
725  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
726  SDLoc dl(N);
727 
728  SDValue InOp = N->getOperand(0);
729  EVT InVT = InOp.getValueType();
730 
731  // Handle some special cases efficiently.
732  switch (getTypeAction(InVT)) {
739  break;
742  // A scalar to vector conversion, where the scalar needs expansion.
743  // If the vector is being split in two then we can just convert the
744  // expanded pieces.
745  if (LoVT == HiVT) {
746  GetExpandedOp(InOp, Lo, Hi);
747  if (DAG.getDataLayout().isBigEndian())
748  std::swap(Lo, Hi);
749  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
750  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
751  return;
752  }
753  break;
755  // If the input is a vector that needs to be split, convert each split
756  // piece of the input now.
757  GetSplitVector(InOp, Lo, Hi);
758  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
759  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
760  return;
761  }
762 
763  // In the general case, convert the input to an integer and split it by hand.
764  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
765  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
766  if (DAG.getDataLayout().isBigEndian())
767  std::swap(LoIntVT, HiIntVT);
768 
769  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
770 
771  if (DAG.getDataLayout().isBigEndian())
772  std::swap(Lo, Hi);
773  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
774  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
775 }
776 
777 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
778  SDValue &Hi) {
779  EVT LoVT, HiVT;
780  SDLoc dl(N);
781  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
782  unsigned LoNumElts = LoVT.getVectorNumElements();
783  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
784  Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, LoOps);
785 
786  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
787  Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, HiOps);
788 }
789 
790 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
791  SDValue &Hi) {
792  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
793  SDLoc dl(N);
794  unsigned NumSubvectors = N->getNumOperands() / 2;
795  if (NumSubvectors == 1) {
796  Lo = N->getOperand(0);
797  Hi = N->getOperand(1);
798  return;
799  }
800 
801  EVT LoVT, HiVT;
802  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
803 
804  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
805  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
806 
807  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
808  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
809 }
810 
811 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
812  SDValue &Hi) {
813  SDValue Vec = N->getOperand(0);
814  SDValue Idx = N->getOperand(1);
815  SDLoc dl(N);
816 
817  EVT LoVT, HiVT;
818  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
819 
820  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
821  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
822  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
823  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
824  TLI.getVectorIdxTy(DAG.getDataLayout())));
825 }
826 
827 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
828  SDValue &Hi) {
829  SDValue Vec = N->getOperand(0);
830  SDValue SubVec = N->getOperand(1);
831  SDValue Idx = N->getOperand(2);
832  SDLoc dl(N);
833  GetSplitVector(Vec, Lo, Hi);
834 
835  EVT VecVT = Vec.getValueType();
836  unsigned VecElems = VecVT.getVectorNumElements();
837  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
838 
839  // If we know the index is 0, and we know the subvector doesn't cross the
840  // boundary between the halves, we can avoid spilling the vector, and insert
841  // into the lower half of the split vector directly.
842  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
843  // the index is constant and there is no boundary crossing. But those cases
844  // don't seem to get hit in practice.
845  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
846  unsigned IdxVal = ConstIdx->getZExtValue();
847  if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
848  EVT LoVT, HiVT;
849  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
850  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
851  return;
852  }
853  }
854 
855  // Spill the vector to the stack.
856  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
857  SDValue Store =
858  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
859 
860  // Store the new subvector into the specified index.
861  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
862  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
863  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
864  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
865 
866  // Load the Lo part from the stack slot.
867  Lo =
868  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
869 
870  // Increment the pointer to the other part.
871  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
872  StackPtr =
873  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
874  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
875 
876  // Load the Hi part from the stack slot.
877  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
878  MinAlign(Alignment, IncrementSize));
879 }
880 
881 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
882  SDValue &Hi) {
883  SDLoc dl(N);
884  GetSplitVector(N->getOperand(0), Lo, Hi);
885  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
886  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
887 }
888 
889 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
890  SDValue &Hi) {
891  SDValue LHSLo, LHSHi;
892  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
893  SDLoc DL(N);
894 
895  SDValue RHSLo, RHSHi;
896  SDValue RHS = N->getOperand(1);
897  EVT RHSVT = RHS.getValueType();
898  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
899  GetSplitVector(RHS, RHSLo, RHSHi);
900  else
901  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
902 
903 
904  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
905  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
906 }
907 
908 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
909  SDValue &Hi) {
910  SDValue LHSLo, LHSHi;
911  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
912  SDLoc dl(N);
913 
914  EVT LoVT, HiVT;
915  std::tie(LoVT, HiVT) =
916  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
917 
918  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
919  DAG.getValueType(LoVT));
920  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
921  DAG.getValueType(HiVT));
922 }
923 
924 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
925  SDValue &Hi) {
926  unsigned Opcode = N->getOpcode();
927  SDValue N0 = N->getOperand(0);
928 
929  SDLoc dl(N);
930  SDValue InLo, InHi;
931  GetSplitVector(N0, InLo, InHi);
932  EVT InLoVT = InLo.getValueType();
933  unsigned InNumElements = InLoVT.getVectorNumElements();
934 
935  EVT OutLoVT, OutHiVT;
936  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
937  unsigned OutNumElements = OutLoVT.getVectorNumElements();
938  assert((2 * OutNumElements) <= InNumElements &&
939  "Illegal extend vector in reg split");
940 
941  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
942  // input vector (i.e. we only use InLo):
943  // OutLo will extend the first OutNumElements from InLo.
944  // OutHi will extend the next OutNumElements from InLo.
945 
946  // Shuffle the elements from InLo for OutHi into the bottom elements to
947  // create a 'fake' InHi.
948  SmallVector<int, 8> SplitHi(InNumElements, -1);
949  for (unsigned i = 0; i != OutNumElements; ++i)
950  SplitHi[i] = i + OutNumElements;
951  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
952 
953  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
954  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
955 }
956 
957 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
958  SDValue &Hi) {
959  SDValue Vec = N->getOperand(0);
960  SDValue Elt = N->getOperand(1);
961  SDValue Idx = N->getOperand(2);
962  SDLoc dl(N);
963  GetSplitVector(Vec, Lo, Hi);
964 
965  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
966  unsigned IdxVal = CIdx->getZExtValue();
967  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
968  if (IdxVal < LoNumElts)
969  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
970  Lo.getValueType(), Lo, Elt, Idx);
971  else
972  Hi =
973  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
974  DAG.getConstant(IdxVal - LoNumElts, dl,
975  TLI.getVectorIdxTy(DAG.getDataLayout())));
976  return;
977  }
978 
979  // See if the target wants to custom expand this node.
980  if (CustomLowerNode(N, N->getValueType(0), true))
981  return;
982 
983  // Spill the vector to the stack.
984  EVT VecVT = Vec.getValueType();
985  EVT EltVT = VecVT.getVectorElementType();
986  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
987  SDValue Store =
988  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
989 
990  // Store the new element. This may be larger than the vector element type,
991  // so use a truncating store.
992  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
993  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
994  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
995  Store =
996  DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT);
997 
998  // Load the Lo part from the stack slot.
999  Lo =
1000  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1001 
1002  // Increment the pointer to the other part.
1003  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1004  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1005  DAG.getConstant(IncrementSize, dl,
1006  StackPtr.getValueType()));
1007 
1008  // Load the Hi part from the stack slot.
1009  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1010  MinAlign(Alignment, IncrementSize));
1011 }
1012 
1013 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1014  SDValue &Hi) {
1015  EVT LoVT, HiVT;
1016  SDLoc dl(N);
1017  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1018  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1019  Hi = DAG.getUNDEF(HiVT);
1020 }
1021 
1022 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1023  SDValue &Hi) {
1024  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1025  EVT LoVT, HiVT;
1026  SDLoc dl(LD);
1027  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1028 
1029  ISD::LoadExtType ExtType = LD->getExtensionType();
1030  SDValue Ch = LD->getChain();
1031  SDValue Ptr = LD->getBasePtr();
1032  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1033  EVT MemoryVT = LD->getMemoryVT();
1034  unsigned Alignment = LD->getOriginalAlignment();
1035  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1036  AAMDNodes AAInfo = LD->getAAInfo();
1037 
1038  EVT LoMemVT, HiMemVT;
1039  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1040 
1041  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1042  LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1043 
1044  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1045  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1046  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
1047  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1048  LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1049  Alignment, MMOFlags, AAInfo);
1050 
1051  // Build a factor node to remember that this load is independent of the
1052  // other one.
1053  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1054  Hi.getValue(1));
1055 
1056  // Legalize the chain result - switch anything that used the old chain to
1057  // use the new one.
1058  ReplaceValueWith(SDValue(LD, 1), Ch);
1059 }
1060 
1061 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1062  SDValue &Lo, SDValue &Hi) {
1063  EVT LoVT, HiVT;
1064  SDLoc dl(MLD);
1065  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1066 
1067  SDValue Ch = MLD->getChain();
1068  SDValue Ptr = MLD->getBasePtr();
1069  SDValue Mask = MLD->getMask();
1070  SDValue Src0 = MLD->getSrc0();
1071  unsigned Alignment = MLD->getOriginalAlignment();
1072  ISD::LoadExtType ExtType = MLD->getExtensionType();
1073 
1074  // if Alignment is equal to the vector size,
1075  // take the half of it for the second part
1076  unsigned SecondHalfAlignment =
1077  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1078  Alignment/2 : Alignment;
1079 
1080  // Split Mask operand
1081  SDValue MaskLo, MaskHi;
1082  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1083  GetSplitVector(Mask, MaskLo, MaskHi);
1084  else
1085  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1086 
1087  EVT MemoryVT = MLD->getMemoryVT();
1088  EVT LoMemVT, HiMemVT;
1089  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1090 
1091  SDValue Src0Lo, Src0Hi;
1092  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1093  GetSplitVector(Src0, Src0Lo, Src0Hi);
1094  else
1095  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1096 
1098  getMachineMemOperand(MLD->getPointerInfo(),
1100  Alignment, MLD->getAAInfo(), MLD->getRanges());
1101 
1102  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
1103  ExtType, MLD->isExpandingLoad());
1104 
1105  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1106  MLD->isExpandingLoad());
1107 
1108  MMO = DAG.getMachineFunction().
1109  getMachineMemOperand(MLD->getPointerInfo(),
1111  SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());
1112 
1113  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
1114  ExtType, MLD->isExpandingLoad());
1115 
1116 
1117  // Build a factor node to remember that this load is independent of the
1118  // other one.
1119  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1120  Hi.getValue(1));
1121 
1122  // Legalize the chain result - switch anything that used the old chain to
1123  // use the new one.
1124  ReplaceValueWith(SDValue(MLD, 1), Ch);
1125 
1126 }
1127 
1128 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1129  SDValue &Lo, SDValue &Hi) {
1130  EVT LoVT, HiVT;
1131  SDLoc dl(MGT);
1132  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1133 
1134  SDValue Ch = MGT->getChain();
1135  SDValue Ptr = MGT->getBasePtr();
1136  SDValue Mask = MGT->getMask();
1137  SDValue Src0 = MGT->getValue();
1138  SDValue Index = MGT->getIndex();
1139  unsigned Alignment = MGT->getOriginalAlignment();
1140 
1141  // Split Mask operand
1142  SDValue MaskLo, MaskHi;
1143  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1144  GetSplitVector(Mask, MaskLo, MaskHi);
1145  else
1146  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1147 
1148  EVT MemoryVT = MGT->getMemoryVT();
1149  EVT LoMemVT, HiMemVT;
1150  // Split MemoryVT
1151  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1152 
1153  SDValue Src0Lo, Src0Hi;
1154  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1155  GetSplitVector(Src0, Src0Lo, Src0Hi);
1156  else
1157  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1158 
1159  SDValue IndexHi, IndexLo;
1160  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1161  GetSplitVector(Index, IndexLo, IndexHi);
1162  else
1163  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1164 
1166  getMachineMemOperand(MGT->getPointerInfo(),
1168  Alignment, MGT->getAAInfo(), MGT->getRanges());
1169 
1170  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
1171  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1172  MMO);
1173 
1174  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
1175  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1176  MMO);
1177 
1178  // Build a factor node to remember that this load is independent of the
1179  // other one.
1180  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1181  Hi.getValue(1));
1182 
1183  // Legalize the chain result - switch anything that used the old chain to
1184  // use the new one.
1185  ReplaceValueWith(SDValue(MGT, 1), Ch);
1186 }
1187 
1188 
1189 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1190  assert(N->getValueType(0).isVector() &&
1191  N->getOperand(0).getValueType().isVector() &&
1192  "Operand types must be vectors");
1193 
1194  EVT LoVT, HiVT;
1195  SDLoc DL(N);
1196  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1197 
1198  // Split the input.
1199  SDValue LL, LH, RL, RH;
1200  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1201  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1202 
1203  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1204  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1205 }
1206 
1207 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1208  SDValue &Hi) {
1209  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1210  EVT LoVT, HiVT;
1211  SDLoc dl(N);
1212  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1213 
1214  // If the input also splits, handle it directly for a compile time speedup.
1215  // Otherwise split it by hand.
1216  EVT InVT = N->getOperand(0).getValueType();
1217  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1218  GetSplitVector(N->getOperand(0), Lo, Hi);
1219  else
1220  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1221 
1222  if (N->getOpcode() == ISD::FP_ROUND) {
1223  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1224  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1225  } else {
1226  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1227  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1228  }
1229 }
1230 
1231 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1232  SDValue &Hi) {
1233  SDLoc dl(N);
1234  EVT SrcVT = N->getOperand(0).getValueType();
1235  EVT DestVT = N->getValueType(0);
1236  EVT LoVT, HiVT;
1237  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1238 
1239  // We can do better than a generic split operation if the extend is doing
1240  // more than just doubling the width of the elements and the following are
1241  // true:
1242  // - The number of vector elements is even,
1243  // - the source type is legal,
1244  // - the type of a split source is illegal,
1245  // - the type of an extended (by doubling element size) source is legal, and
1246  // - the type of that extended source when split is legal.
1247  //
1248  // This won't necessarily completely legalize the operation, but it will
1249  // more effectively move in the right direction and prevent falling down
1250  // to scalarization in many cases due to the input vector being split too
1251  // far.
1252  unsigned NumElements = SrcVT.getVectorNumElements();
1253  if ((NumElements & 1) == 0 &&
1254  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1255  LLVMContext &Ctx = *DAG.getContext();
1256  EVT NewSrcVT = EVT::getVectorVT(
1257  Ctx, EVT::getIntegerVT(
1258  Ctx, SrcVT.getScalarSizeInBits() * 2),
1259  NumElements);
1260  EVT SplitSrcVT =
1261  EVT::getVectorVT(Ctx, SrcVT.getVectorElementType(), NumElements / 2);
1262  EVT SplitLoVT, SplitHiVT;
1263  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1264  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1265  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1266  DEBUG(dbgs() << "Split vector extend via incremental extend:";
1267  N->dump(&DAG); dbgs() << "\n");
1268  // Extend the source vector by one step.
1269  SDValue NewSrc =
1270  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1271  // Get the low and high halves of the new, extended one step, vector.
1272  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1273  // Extend those vector halves the rest of the way.
1274  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1275  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1276  return;
1277  }
1278  }
1279  // Fall back to the generic unary operator splitting otherwise.
1280  SplitVecRes_UnaryOp(N, Lo, Hi);
1281 }
1282 
1283 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1284  SDValue &Lo, SDValue &Hi) {
1285  // The low and high parts of the original input give four input vectors.
1286  SDValue Inputs[4];
1287  SDLoc dl(N);
1288  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1289  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1290  EVT NewVT = Inputs[0].getValueType();
1291  unsigned NewElts = NewVT.getVectorNumElements();
1292 
1293  // If Lo or Hi uses elements from at most two of the four input vectors, then
1294  // express it as a vector shuffle of those two inputs. Otherwise extract the
1295  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1297  for (unsigned High = 0; High < 2; ++High) {
1298  SDValue &Output = High ? Hi : Lo;
1299 
1300  // Build a shuffle mask for the output, discovering on the fly which
1301  // input vectors to use as shuffle operands (recorded in InputUsed).
1302  // If building a suitable shuffle vector proves too hard, then bail
1303  // out with useBuildVector set.
1304  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1305  unsigned FirstMaskIdx = High * NewElts;
1306  bool useBuildVector = false;
1307  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1308  // The mask element. This indexes into the input.
1309  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1310 
1311  // The input vector this mask element indexes into.
1312  unsigned Input = (unsigned)Idx / NewElts;
1313 
1314  if (Input >= array_lengthof(Inputs)) {
1315  // The mask element does not index into any input vector.
1316  Ops.push_back(-1);
1317  continue;
1318  }
1319 
1320  // Turn the index into an offset from the start of the input vector.
1321  Idx -= Input * NewElts;
1322 
1323  // Find or create a shuffle vector operand to hold this input.
1324  unsigned OpNo;
1325  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1326  if (InputUsed[OpNo] == Input) {
1327  // This input vector is already an operand.
1328  break;
1329  } else if (InputUsed[OpNo] == -1U) {
1330  // Create a new operand for this input vector.
1331  InputUsed[OpNo] = Input;
1332  break;
1333  }
1334  }
1335 
1336  if (OpNo >= array_lengthof(InputUsed)) {
1337  // More than two input vectors used! Give up on trying to create a
1338  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1339  useBuildVector = true;
1340  break;
1341  }
1342 
1343  // Add the mask index for the new shuffle vector.
1344  Ops.push_back(Idx + OpNo * NewElts);
1345  }
1346 
1347  if (useBuildVector) {
1348  EVT EltVT = NewVT.getVectorElementType();
1350 
1351  // Extract the input elements by hand.
1352  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1353  // The mask element. This indexes into the input.
1354  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1355 
1356  // The input vector this mask element indexes into.
1357  unsigned Input = (unsigned)Idx / NewElts;
1358 
1359  if (Input >= array_lengthof(Inputs)) {
1360  // The mask element is "undef" or indexes off the end of the input.
1361  SVOps.push_back(DAG.getUNDEF(EltVT));
1362  continue;
1363  }
1364 
1365  // Turn the index into an offset from the start of the input vector.
1366  Idx -= Input * NewElts;
1367 
1368  // Extract the vector element by hand.
1369  SVOps.push_back(DAG.getNode(
1370  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1371  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1372  }
1373 
1374  // Construct the Lo/Hi output using a BUILD_VECTOR.
1375  Output = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, SVOps);
1376  } else if (InputUsed[0] == -1U) {
1377  // No input vectors were used! The result is undefined.
1378  Output = DAG.getUNDEF(NewVT);
1379  } else {
1380  SDValue Op0 = Inputs[InputUsed[0]];
1381  // If only one input was used, use an undefined vector for the other.
1382  SDValue Op1 = InputUsed[1] == -1U ?
1383  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1384  // At least one input vector was used. Create a new shuffle vector.
1385  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1386  }
1387 
1388  Ops.clear();
1389  }
1390 }
1391 
1392 
1393 //===----------------------------------------------------------------------===//
1394 // Operand Vector Splitting
1395 //===----------------------------------------------------------------------===//
1396 
1397 /// This method is called when the specified operand of the specified node is
1398 /// found to need vector splitting. At this point, all of the result types of
1399 /// the node are known to be legal, but other operands of the node may need
1400 /// legalization as well as the specified one.
1401 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1402  DEBUG(dbgs() << "Split node operand: ";
1403  N->dump(&DAG);
1404  dbgs() << "\n");
1405  SDValue Res = SDValue();
1406 
1407  // See if the target wants to custom split this node.
1408  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1409  return false;
1410 
1411  if (!Res.getNode()) {
1412  switch (N->getOpcode()) {
1413  default:
1414 #ifndef NDEBUG
1415  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1416  N->dump(&DAG);
1417  dbgs() << "\n";
1418 #endif
1419  report_fatal_error("Do not know how to split this operator's "
1420  "operand!\n");
1421 
1422  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1423  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1424  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1425  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1426  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1427  case ISD::TRUNCATE:
1428  Res = SplitVecOp_TruncateHelper(N);
1429  break;
1430  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1431  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1432  case ISD::STORE:
1433  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1434  break;
1435  case ISD::MSTORE:
1436  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1437  break;
1438  case ISD::MSCATTER:
1439  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1440  break;
1441  case ISD::MGATHER:
1442  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1443  break;
1444  case ISD::VSELECT:
1445  Res = SplitVecOp_VSELECT(N, OpNo);
1446  break;
1447  case ISD::FP_TO_SINT:
1448  case ISD::FP_TO_UINT:
1449  if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
1450  Res = SplitVecOp_TruncateHelper(N);
1451  else
1452  Res = SplitVecOp_UnaryOp(N);
1453  break;
1454  case ISD::SINT_TO_FP:
1455  case ISD::UINT_TO_FP:
1456  if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
1457  Res = SplitVecOp_TruncateHelper(N);
1458  else
1459  Res = SplitVecOp_UnaryOp(N);
1460  break;
1461  case ISD::CTTZ:
1462  case ISD::CTLZ:
1463  case ISD::CTPOP:
1464  case ISD::FP_EXTEND:
1465  case ISD::SIGN_EXTEND:
1466  case ISD::ZERO_EXTEND:
1467  case ISD::ANY_EXTEND:
1468  case ISD::FTRUNC:
1469  Res = SplitVecOp_UnaryOp(N);
1470  break;
1471  }
1472  }
1473 
1474  // If the result is null, the sub-method took care of registering results etc.
1475  if (!Res.getNode()) return false;
1476 
1477  // If the result is N, the sub-method updated N in place. Tell the legalizer
1478  // core about this.
1479  if (Res.getNode() == N)
1480  return true;
1481 
1482  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1483  "Invalid operand expansion");
1484 
1485  ReplaceValueWith(SDValue(N, 0), Res);
1486  return false;
1487 }
1488 
1489 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1490  // The only possibility for an illegal operand is the mask, since result type
1491  // legalization would have handled this node already otherwise.
1492  assert(OpNo == 0 && "Illegal operand must be mask");
1493 
1494  SDValue Mask = N->getOperand(0);
1495  SDValue Src0 = N->getOperand(1);
1496  SDValue Src1 = N->getOperand(2);
1497  EVT Src0VT = Src0.getValueType();
1498  SDLoc DL(N);
1499  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1500 
1501  SDValue Lo, Hi;
1502  GetSplitVector(N->getOperand(0), Lo, Hi);
1503  assert(Lo.getValueType() == Hi.getValueType() &&
1504  "Lo and Hi have differing types");
1505 
1506  EVT LoOpVT, HiOpVT;
1507  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1508  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1509 
1510  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1511  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1512  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1513  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1514 
1515  SDValue LoSelect =
1516  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1517  SDValue HiSelect =
1518  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1519 
1520  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1521 }
1522 
1523 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1524  // The result has a legal vector type, but the input needs splitting.
1525  EVT ResVT = N->getValueType(0);
1526  SDValue Lo, Hi;
1527  SDLoc dl(N);
1528  GetSplitVector(N->getOperand(0), Lo, Hi);
1529  EVT InVT = Lo.getValueType();
1530 
1531  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1532  InVT.getVectorNumElements());
1533 
1534  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1535  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1536 
1537  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1538 }
1539 
1540 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1541  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1542  // end up being split all the way down to individual components. Convert the
1543  // split pieces into integers and reassemble.
1544  SDValue Lo, Hi;
1545  GetSplitVector(N->getOperand(0), Lo, Hi);
1546  Lo = BitConvertToInteger(Lo);
1547  Hi = BitConvertToInteger(Hi);
1548 
1549  if (DAG.getDataLayout().isBigEndian())
1550  std::swap(Lo, Hi);
1551 
1552  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1553  JoinIntegers(Lo, Hi));
1554 }
1555 
1556 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1557  // We know that the extracted result type is legal.
1558  EVT SubVT = N->getValueType(0);
1559  SDValue Idx = N->getOperand(1);
1560  SDLoc dl(N);
1561  SDValue Lo, Hi;
1562  GetSplitVector(N->getOperand(0), Lo, Hi);
1563 
1564  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1565  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1566 
1567  if (IdxVal < LoElts) {
1568  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1569  "Extracted subvector crosses vector split!");
1570  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1571  } else {
1572  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1573  DAG.getConstant(IdxVal - LoElts, dl,
1574  Idx.getValueType()));
1575  }
1576 }
1577 
1578 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1579  SDValue Vec = N->getOperand(0);
1580  SDValue Idx = N->getOperand(1);
1581  EVT VecVT = Vec.getValueType();
1582 
1583  if (isa<ConstantSDNode>(Idx)) {
1584  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1585  assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1586 
1587  SDValue Lo, Hi;
1588  GetSplitVector(Vec, Lo, Hi);
1589 
1590  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1591 
1592  if (IdxVal < LoElts)
1593  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1594  return SDValue(DAG.UpdateNodeOperands(N, Hi,
1595  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
1596  Idx.getValueType())), 0);
1597  }
1598 
1599  // See if the target wants to custom expand this node.
1600  if (CustomLowerNode(N, N->getValueType(0), true))
1601  return SDValue();
1602 
1603  // Make the vector elements byte-addressable if they aren't already.
1604  SDLoc dl(N);
1605  EVT EltVT = VecVT.getVectorElementType();
1606  if (EltVT.getSizeInBits() < 8) {
1607  SmallVector<SDValue, 4> ElementOps;
1608  for (unsigned i = 0; i < VecVT.getVectorNumElements(); ++i) {
1609  ElementOps.push_back(DAG.getAnyExtOrTrunc(
1610  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vec,
1611  DAG.getConstant(i, dl, MVT::i8)),
1612  dl, MVT::i8));
1613  }
1614 
1615  EltVT = MVT::i8;
1616  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1617  VecVT.getVectorNumElements());
1618  Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, ElementOps);
1619  }
1620 
1621  // Store the vector to the stack.
1622  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1623  SDValue Store =
1624  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1625 
1626  // Load back the required element.
1627  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1628  return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1629  MachinePointerInfo(), EltVT);
1630 }
1631 
1632 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
1633  unsigned OpNo) {
1634  EVT LoVT, HiVT;
1635  SDLoc dl(MGT);
1636  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1637 
1638  SDValue Ch = MGT->getChain();
1639  SDValue Ptr = MGT->getBasePtr();
1640  SDValue Index = MGT->getIndex();
1641  SDValue Mask = MGT->getMask();
1642  SDValue Src0 = MGT->getValue();
1643  unsigned Alignment = MGT->getOriginalAlignment();
1644 
1645  SDValue MaskLo, MaskHi;
1646  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1647  // Split Mask operand
1648  GetSplitVector(Mask, MaskLo, MaskHi);
1649  else
1650  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1651 
1652  EVT MemoryVT = MGT->getMemoryVT();
1653  EVT LoMemVT, HiMemVT;
1654  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1655 
1656  SDValue Src0Lo, Src0Hi;
1657  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1658  GetSplitVector(Src0, Src0Lo, Src0Hi);
1659  else
1660  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1661 
1662  SDValue IndexHi, IndexLo;
1663  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1664  GetSplitVector(Index, IndexLo, IndexHi);
1665  else
1666  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1667 
1669  getMachineMemOperand(MGT->getPointerInfo(),
1671  Alignment, MGT->getAAInfo(), MGT->getRanges());
1672 
1673  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
1674  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
1675  OpsLo, MMO);
1676 
1677  MMO = DAG.getMachineFunction().
1678  getMachineMemOperand(MGT->getPointerInfo(),
1680  Alignment, MGT->getAAInfo(),
1681  MGT->getRanges());
1682 
1683  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
1684  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
1685  OpsHi, MMO);
1686 
1687  // Build a factor node to remember that this load is independent of the
1688  // other one.
1689  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1690  Hi.getValue(1));
1691 
1692  // Legalize the chain result - switch anything that used the old chain to
1693  // use the new one.
1694  ReplaceValueWith(SDValue(MGT, 1), Ch);
1695 
1696  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
1697  Hi);
1698  ReplaceValueWith(SDValue(MGT, 0), Res);
1699  return SDValue();
1700 }
1701 
1702 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
1703  unsigned OpNo) {
1704  SDValue Ch = N->getChain();
1705  SDValue Ptr = N->getBasePtr();
1706  SDValue Mask = N->getMask();
1707  SDValue Data = N->getValue();
1708  EVT MemoryVT = N->getMemoryVT();
1709  unsigned Alignment = N->getOriginalAlignment();
1710  SDLoc DL(N);
1711 
1712  EVT LoMemVT, HiMemVT;
1713  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1714 
1715  SDValue DataLo, DataHi;
1716  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1717  // Split Data operand
1718  GetSplitVector(Data, DataLo, DataHi);
1719  else
1720  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1721 
1722  SDValue MaskLo, MaskHi;
1723  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1724  // Split Mask operand
1725  GetSplitVector(Mask, MaskLo, MaskHi);
1726  else
1727  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1728 
1729  MaskLo = PromoteTargetBoolean(MaskLo, DataLo.getValueType());
1730  MaskHi = PromoteTargetBoolean(MaskHi, DataHi.getValueType());
1731 
1732  // if Alignment is equal to the vector size,
1733  // take the half of it for the second part
1734  unsigned SecondHalfAlignment =
1735  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
1736  Alignment/2 : Alignment;
1737 
1738  SDValue Lo, Hi;
1740  getMachineMemOperand(N->getPointerInfo(),
1742  Alignment, N->getAAInfo(), N->getRanges());
1743 
1744  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
1745  N->isTruncatingStore(),
1746  N->isCompressingStore());
1747 
1748  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
1749  N->isCompressingStore());
1750  MMO = DAG.getMachineFunction().
1751  getMachineMemOperand(N->getPointerInfo(),
1753  SecondHalfAlignment, N->getAAInfo(), N->getRanges());
1754 
1755  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
1757 
1758  // Build a factor node to remember that this store is independent of the
1759  // other one.
1760  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1761 }
1762 
1763 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
1764  unsigned OpNo) {
1765  SDValue Ch = N->getChain();
1766  SDValue Ptr = N->getBasePtr();
1767  SDValue Mask = N->getMask();
1768  SDValue Index = N->getIndex();
1769  SDValue Data = N->getValue();
1770  EVT MemoryVT = N->getMemoryVT();
1771  unsigned Alignment = N->getOriginalAlignment();
1772  SDLoc DL(N);
1773 
1774  // Split all operands
1775  EVT LoMemVT, HiMemVT;
1776  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1777 
1778  SDValue DataLo, DataHi;
1779  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1780  // Split Data operand
1781  GetSplitVector(Data, DataLo, DataHi);
1782  else
1783  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1784 
1785  SDValue MaskLo, MaskHi;
1786  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1787  // Split Mask operand
1788  GetSplitVector(Mask, MaskLo, MaskHi);
1789  else
1790  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1791 
1792  SDValue IndexHi, IndexLo;
1793  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1794  GetSplitVector(Index, IndexLo, IndexHi);
1795  else
1796  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
1797 
1798  SDValue Lo, Hi;
1800  getMachineMemOperand(N->getPointerInfo(),
1802  Alignment, N->getAAInfo(), N->getRanges());
1803 
1804  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo};
1805  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
1806  DL, OpsLo, MMO);
1807 
1808  MMO = DAG.getMachineFunction().
1809  getMachineMemOperand(N->getPointerInfo(),
1811  Alignment, N->getAAInfo(), N->getRanges());
1812 
1813  SDValue OpsHi[] = {Ch, DataHi, MaskHi, Ptr, IndexHi};
1814  Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
1815  DL, OpsHi, MMO);
1816 
1817  // Build a factor node to remember that this store is independent of the
1818  // other one.
1819  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1820 }
1821 
1822 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
1823  assert(N->isUnindexed() && "Indexed store of vector?");
1824  assert(OpNo == 1 && "Can only split the stored value");
1825  SDLoc DL(N);
1826 
1827  bool isTruncating = N->isTruncatingStore();
1828  SDValue Ch = N->getChain();
1829  SDValue Ptr = N->getBasePtr();
1830  EVT MemoryVT = N->getMemoryVT();
1831  unsigned Alignment = N->getOriginalAlignment();
1832  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
1833  AAMDNodes AAInfo = N->getAAInfo();
1834  SDValue Lo, Hi;
1835  GetSplitVector(N->getOperand(1), Lo, Hi);
1836 
1837  EVT LoMemVT, HiMemVT;
1838  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1839 
1840  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1841 
1842  if (isTruncating)
1843  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
1844  Alignment, MMOFlags, AAInfo);
1845  else
1846  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
1847  AAInfo);
1848 
1849  // Increment the pointer to the other half.
1850  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
1851  DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
1852 
1853  if (isTruncating)
1854  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
1855  N->getPointerInfo().getWithOffset(IncrementSize),
1856  HiMemVT, Alignment, MMOFlags, AAInfo);
1857  else
1858  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
1859  N->getPointerInfo().getWithOffset(IncrementSize),
1860  Alignment, MMOFlags, AAInfo);
1861 
1862  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1863 }
1864 
1865 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
1866  SDLoc DL(N);
1867 
1868  // The input operands all must have the same type, and we know the result
1869  // type is valid. Convert this to a buildvector which extracts all the
1870  // input elements.
1871  // TODO: If the input elements are power-two vectors, we could convert this to
1872  // a new CONCAT_VECTORS node with elements that are half-wide.
1874  EVT EltVT = N->getValueType(0).getVectorElementType();
1875  for (const SDValue &Op : N->op_values()) {
1876  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
1877  i != e; ++i) {
1878  Elts.push_back(DAG.getNode(
1879  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
1880  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1881  }
1882  }
1883 
1884  return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts);
1885 }
1886 
1887 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
1888  // The result type is legal, but the input type is illegal. If splitting
1889  // ends up with the result type of each half still being legal, just
1890  // do that. If, however, that would result in an illegal result type,
1891  // we can try to get more clever with power-two vectors. Specifically,
1892  // split the input type, but also widen the result element size, then
1893  // concatenate the halves and truncate again. For example, consider a target
1894  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
1895  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
1896  // %inlo = v4i32 extract_subvector %in, 0
1897  // %inhi = v4i32 extract_subvector %in, 4
1898  // %lo16 = v4i16 trunc v4i32 %inlo
1899  // %hi16 = v4i16 trunc v4i32 %inhi
1900  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
1901  // %res = v8i8 trunc v8i16 %in16
1902  //
1903  // Without this transform, the original truncate would end up being
1904  // scalarized, which is pretty much always a last resort.
1905  SDValue InVec = N->getOperand(0);
1906  EVT InVT = InVec->getValueType(0);
1907  EVT OutVT = N->getValueType(0);
1908  unsigned NumElements = OutVT.getVectorNumElements();
1909  bool IsFloat = OutVT.isFloatingPoint();
1910 
1911  // Widening should have already made sure this is a power-two vector
1912  // if we're trying to split it at all. assert() that's true, just in case.
1913  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
1914 
1915  unsigned InElementSize = InVT.getScalarSizeInBits();
1916  unsigned OutElementSize = OutVT.getScalarSizeInBits();
1917 
1918  // If the input elements are only 1/2 the width of the result elements,
1919  // just use the normal splitting. Our trick only work if there's room
1920  // to split more than once.
1921  if (InElementSize <= OutElementSize * 2)
1922  return SplitVecOp_UnaryOp(N);
1923  SDLoc DL(N);
1924 
1925  // Extract the halves of the input via extract_subvector.
1926  SDValue InLoVec, InHiVec;
1927  std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL);
1928  // Truncate them to 1/2 the element size.
1929  EVT HalfElementVT = IsFloat ?
1930  EVT::getFloatingPointVT(InElementSize/2) :
1931  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
1932  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
1933  NumElements/2);
1934  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
1935  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
1936  // Concatenate them to get the full intermediate truncation result.
1937  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
1938  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
1939  HalfHi);
1940  // Now finish up by truncating all the way down to the original result
1941  // type. This should normally be something that ends up being legal directly,
1942  // but in theory if a target has very wide vectors and an annoyingly
1943  // restricted set of legal types, this split can chain to build things up.
1944  return IsFloat
1945  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
1946  DAG.getTargetConstant(
1947  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
1948  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
1949 }
1950 
1951 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
1952  assert(N->getValueType(0).isVector() &&
1953  N->getOperand(0).getValueType().isVector() &&
1954  "Operand types must be vectors");
1955  // The result has a legal vector type, but the input needs splitting.
1956  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
1957  SDLoc DL(N);
1958  GetSplitVector(N->getOperand(0), Lo0, Hi0);
1959  GetSplitVector(N->getOperand(1), Lo1, Hi1);
1960  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
1961  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
1962  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
1963 
1964  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
1965  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
1966  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
1967  return PromoteTargetBoolean(Con, N->getValueType(0));
1968 }
1969 
1970 
1971 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
1972  // The result has a legal vector type, but the input needs splitting.
1973  EVT ResVT = N->getValueType(0);
1974  SDValue Lo, Hi;
1975  SDLoc DL(N);
1976  GetSplitVector(N->getOperand(0), Lo, Hi);
1977  EVT InVT = Lo.getValueType();
1978 
1979  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1980  InVT.getVectorNumElements());
1981 
1982  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
1983  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
1984 
1985  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
1986 }
1987 
1988 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
1989  // The result (and the first input) has a legal vector type, but the second
1990  // input needs splitting.
1991  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
1992 }
1993 
1994 
1995 //===----------------------------------------------------------------------===//
1996 // Result Vector Widening
1997 //===----------------------------------------------------------------------===//
1998 
1999 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2000  DEBUG(dbgs() << "Widen node result " << ResNo << ": ";
2001  N->dump(&DAG);
2002  dbgs() << "\n");
2003 
2004  // See if the target wants to custom widen this node.
2005  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2006  return;
2007 
2008  SDValue Res = SDValue();
2009  switch (N->getOpcode()) {
2010  default:
2011 #ifndef NDEBUG
2012  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2013  N->dump(&DAG);
2014  dbgs() << "\n";
2015 #endif
2016  llvm_unreachable("Do not know how to widen the result of this operator!");
2017 
2018  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2019  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2020  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2021  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2022  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2023  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2024  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2025  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2026  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2027  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2028  case ISD::VSELECT:
2029  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2030  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2031  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2032  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2033  case ISD::VECTOR_SHUFFLE:
2034  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2035  break;
2036  case ISD::MLOAD:
2037  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2038  break;
2039  case ISD::MGATHER:
2040  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2041  break;
2042 
2043  case ISD::ADD:
2044  case ISD::AND:
2045  case ISD::MUL:
2046  case ISD::MULHS:
2047  case ISD::MULHU:
2048  case ISD::OR:
2049  case ISD::SUB:
2050  case ISD::XOR:
2051  case ISD::FMINNUM:
2052  case ISD::FMAXNUM:
2053  case ISD::FMINNAN:
2054  case ISD::FMAXNAN:
2055  case ISD::SMIN:
2056  case ISD::SMAX:
2057  case ISD::UMIN:
2058  case ISD::UMAX:
2059  Res = WidenVecRes_Binary(N);
2060  break;
2061 
2062  case ISD::FADD:
2063  case ISD::FMUL:
2064  case ISD::FPOW:
2065  case ISD::FSUB:
2066  case ISD::FDIV:
2067  case ISD::FREM:
2068  case ISD::SDIV:
2069  case ISD::UDIV:
2070  case ISD::SREM:
2071  case ISD::UREM:
2072  Res = WidenVecRes_BinaryCanTrap(N);
2073  break;
2074 
2075  case ISD::FCOPYSIGN:
2076  Res = WidenVecRes_FCOPYSIGN(N);
2077  break;
2078 
2079  case ISD::FPOWI:
2080  Res = WidenVecRes_POWI(N);
2081  break;
2082 
2083  case ISD::SHL:
2084  case ISD::SRA:
2085  case ISD::SRL:
2086  Res = WidenVecRes_Shift(N);
2087  break;
2088 
2092  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2093  break;
2094 
2095  case ISD::ANY_EXTEND:
2096  case ISD::FP_EXTEND:
2097  case ISD::FP_ROUND:
2098  case ISD::FP_TO_SINT:
2099  case ISD::FP_TO_UINT:
2100  case ISD::SIGN_EXTEND:
2101  case ISD::SINT_TO_FP:
2102  case ISD::TRUNCATE:
2103  case ISD::UINT_TO_FP:
2104  case ISD::ZERO_EXTEND:
2105  Res = WidenVecRes_Convert(N);
2106  break;
2107 
2108  case ISD::BITREVERSE:
2109  case ISD::BSWAP:
2110  case ISD::CTLZ:
2111  case ISD::CTPOP:
2112  case ISD::CTTZ:
2113  case ISD::FABS:
2114  case ISD::FCEIL:
2115  case ISD::FCOS:
2116  case ISD::FEXP:
2117  case ISD::FEXP2:
2118  case ISD::FFLOOR:
2119  case ISD::FLOG:
2120  case ISD::FLOG10:
2121  case ISD::FLOG2:
2122  case ISD::FNEARBYINT:
2123  case ISD::FNEG:
2124  case ISD::FRINT:
2125  case ISD::FROUND:
2126  case ISD::FSIN:
2127  case ISD::FSQRT:
2128  case ISD::FTRUNC:
2129  Res = WidenVecRes_Unary(N);
2130  break;
2131  case ISD::FMA:
2132  Res = WidenVecRes_Ternary(N);
2133  break;
2134  }
2135 
2136  // If Res is null, the sub-method took care of registering the result.
2137  if (Res.getNode())
2138  SetWidenedVector(SDValue(N, ResNo), Res);
2139 }
2140 
2141 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2142  // Ternary op widening.
2143  SDLoc dl(N);
2144  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2145  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2146  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2147  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2148  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2149 }
2150 
2151 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2152  // Binary op widening.
2153  SDLoc dl(N);
2154  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2155  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2156  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2157  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2158 }
2159 
2160 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2161  // Binary op widening for operations that can trap.
2162  unsigned Opcode = N->getOpcode();
2163  SDLoc dl(N);
2164  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2165  EVT WidenEltVT = WidenVT.getVectorElementType();
2166  EVT VT = WidenVT;
2167  unsigned NumElts = VT.getVectorNumElements();
2168  const SDNodeFlags *Flags = N->getFlags();
2169  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2170  NumElts = NumElts / 2;
2171  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2172  }
2173 
2174  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2175  // Operation doesn't trap so just widen as normal.
2176  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2177  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2178  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2179  }
2180 
2181  // No legal vector version so unroll the vector operation and then widen.
2182  if (NumElts == 1)
2183  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2184 
2185  // Since the operation can trap, apply operation on the original vector.
2186  EVT MaxVT = VT;
2187  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2188  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2189  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2190 
2191  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2192  unsigned ConcatEnd = 0; // Current ConcatOps index.
2193  int Idx = 0; // Current Idx into input vectors.
2194 
2195  // NumElts := greatest legal vector size (at most WidenVT)
2196  // while (orig. vector has unhandled elements) {
2197  // take munches of size NumElts from the beginning and add to ConcatOps
2198  // NumElts := next smaller supported vector size or 1
2199  // }
2200  while (CurNumElts != 0) {
2201  while (CurNumElts >= NumElts) {
2202  SDValue EOp1 = DAG.getNode(
2203  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2204  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2205  SDValue EOp2 = DAG.getNode(
2206  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2207  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2208  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2209  Idx += NumElts;
2210  CurNumElts -= NumElts;
2211  }
2212  do {
2213  NumElts = NumElts / 2;
2214  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2215  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2216 
2217  if (NumElts == 1) {
2218  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2219  SDValue EOp1 = DAG.getNode(
2220  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2221  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2222  SDValue EOp2 = DAG.getNode(
2223  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2224  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2225  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2226  EOp1, EOp2, Flags);
2227  }
2228  CurNumElts = 0;
2229  }
2230  }
2231 
2232  // Check to see if we have a single operation with the widen type.
2233  if (ConcatEnd == 1) {
2234  VT = ConcatOps[0].getValueType();
2235  if (VT == WidenVT)
2236  return ConcatOps[0];
2237  }
2238 
2239  // while (Some element of ConcatOps is not of type MaxVT) {
2240  // From the end of ConcatOps, collect elements of the same type and put
2241  // them into an op of the next larger supported type
2242  // }
2243  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2244  Idx = ConcatEnd - 1;
2245  VT = ConcatOps[Idx--].getValueType();
2246  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2247  Idx--;
2248 
2249  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2250  EVT NextVT;
2251  do {
2252  NextSize *= 2;
2253  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2254  } while (!TLI.isTypeLegal(NextVT));
2255 
2256  if (!VT.isVector()) {
2257  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2258  SDValue VecOp = DAG.getUNDEF(NextVT);
2259  unsigned NumToInsert = ConcatEnd - Idx - 1;
2260  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2261  VecOp = DAG.getNode(
2262  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2263  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2264  }
2265  ConcatOps[Idx+1] = VecOp;
2266  ConcatEnd = Idx + 2;
2267  } else {
2268  // Vector type, create a CONCAT_VECTORS of type NextVT
2269  SDValue undefVec = DAG.getUNDEF(VT);
2270  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2271  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2272  unsigned RealVals = ConcatEnd - Idx - 1;
2273  unsigned SubConcatEnd = 0;
2274  unsigned SubConcatIdx = Idx + 1;
2275  while (SubConcatEnd < RealVals)
2276  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2277  while (SubConcatEnd < OpsToConcat)
2278  SubConcatOps[SubConcatEnd++] = undefVec;
2279  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2280  NextVT, SubConcatOps);
2281  ConcatEnd = SubConcatIdx + 1;
2282  }
2283  }
2284 
2285  // Check to see if we have a single operation with the widen type.
2286  if (ConcatEnd == 1) {
2287  VT = ConcatOps[0].getValueType();
2288  if (VT == WidenVT)
2289  return ConcatOps[0];
2290  }
2291 
2292  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2293  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2294  if (NumOps != ConcatEnd ) {
2295  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2296  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2297  ConcatOps[j] = UndefVal;
2298  }
2299  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2300  makeArrayRef(ConcatOps.data(), NumOps));
2301 }
2302 
2303 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
2304  SDValue InOp = N->getOperand(0);
2305  SDLoc DL(N);
2306 
2307  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2308  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2309 
2310  EVT InVT = InOp.getValueType();
2311  EVT InEltVT = InVT.getVectorElementType();
2312  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2313 
2314  unsigned Opcode = N->getOpcode();
2315  unsigned InVTNumElts = InVT.getVectorNumElements();
2316  const SDNodeFlags *Flags = N->getFlags();
2317  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2318  InOp = GetWidenedVector(N->getOperand(0));
2319  InVT = InOp.getValueType();
2320  InVTNumElts = InVT.getVectorNumElements();
2321  if (InVTNumElts == WidenNumElts) {
2322  if (N->getNumOperands() == 1)
2323  return DAG.getNode(Opcode, DL, WidenVT, InOp);
2324  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
2325  }
2326  }
2327 
2328  if (TLI.isTypeLegal(InWidenVT)) {
2329  // Because the result and the input are different vector types, widening
2330  // the result could create a legal type but widening the input might make
2331  // it an illegal type that might lead to repeatedly splitting the input
2332  // and then widening it. To avoid this, we widen the input only if
2333  // it results in a legal type.
2334  if (WidenNumElts % InVTNumElts == 0) {
2335  // Widen the input and call convert on the widened input vector.
2336  unsigned NumConcat = WidenNumElts/InVTNumElts;
2337  SmallVector<SDValue, 16> Ops(NumConcat);
2338  Ops[0] = InOp;
2339  SDValue UndefVal = DAG.getUNDEF(InVT);
2340  for (unsigned i = 1; i != NumConcat; ++i)
2341  Ops[i] = UndefVal;
2342  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
2343  if (N->getNumOperands() == 1)
2344  return DAG.getNode(Opcode, DL, WidenVT, InVec);
2345  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
2346  }
2347 
2348  if (InVTNumElts % WidenNumElts == 0) {
2349  SDValue InVal = DAG.getNode(
2350  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
2351  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2352  // Extract the input and convert the shorten input vector.
2353  if (N->getNumOperands() == 1)
2354  return DAG.getNode(Opcode, DL, WidenVT, InVal);
2355  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
2356  }
2357  }
2358 
2359  // Otherwise unroll into some nasty scalar code and rebuild the vector.
2360  SmallVector<SDValue, 16> Ops(WidenNumElts);
2361  EVT EltVT = WidenVT.getVectorElementType();
2362  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2363  unsigned i;
2364  for (i=0; i < MinElts; ++i) {
2365  SDValue Val = DAG.getNode(
2366  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
2367  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2368  if (N->getNumOperands() == 1)
2369  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
2370  else
2371  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
2372  }
2373 
2374  SDValue UndefVal = DAG.getUNDEF(EltVT);
2375  for (; i < WidenNumElts; ++i)
2376  Ops[i] = UndefVal;
2377 
2378  return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops);
2379 }
2380 
2381 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
2382  unsigned Opcode = N->getOpcode();
2383  SDValue InOp = N->getOperand(0);
2384  SDLoc DL(N);
2385 
2386  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2387  EVT WidenSVT = WidenVT.getVectorElementType();
2388  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2389 
2390  EVT InVT = InOp.getValueType();
2391  EVT InSVT = InVT.getVectorElementType();
2392  unsigned InVTNumElts = InVT.getVectorNumElements();
2393 
2394  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2395  InOp = GetWidenedVector(InOp);
2396  InVT = InOp.getValueType();
2397  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
2398  switch (Opcode) {
2400  return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
2402  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2404  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2405  }
2406  }
2407  }
2408 
2409  // Unroll, extend the scalars and rebuild the vector.
2411  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
2412  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
2413  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2414  switch (Opcode) {
2416  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
2417  break;
2419  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
2420  break;
2422  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
2423  break;
2424  default:
2425  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
2426  }
2427  Ops.push_back(Val);
2428  }
2429 
2430  while (Ops.size() != WidenNumElts)
2431  Ops.push_back(DAG.getUNDEF(WidenSVT));
2432 
2433  return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops);
2434 }
2435 
2436 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
2437  // If this is an FCOPYSIGN with same input types, we can treat it as a
2438  // normal (can trap) binary op.
2439  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
2440  return WidenVecRes_BinaryCanTrap(N);
2441 
2442  // If the types are different, fall back to unrolling.
2443  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2444  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2445 }
2446 
2447 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
2448  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2449  SDValue InOp = GetWidenedVector(N->getOperand(0));
2450  SDValue ShOp = N->getOperand(1);
2451  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2452 }
2453 
2454 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
2455  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2456  SDValue InOp = GetWidenedVector(N->getOperand(0));
2457  SDValue ShOp = N->getOperand(1);
2458 
2459  EVT ShVT = ShOp.getValueType();
2460  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
2461  ShOp = GetWidenedVector(ShOp);
2462  ShVT = ShOp.getValueType();
2463  }
2464  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
2465  ShVT.getVectorElementType(),
2466  WidenVT.getVectorNumElements());
2467  if (ShVT != ShWidenVT)
2468  ShOp = ModifyToType(ShOp, ShWidenVT);
2469 
2470  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2471 }
2472 
2473 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
2474  // Unary op widening.
2475  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2476  SDValue InOp = GetWidenedVector(N->getOperand(0));
2477  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
2478 }
2479 
2480 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
2481  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2482  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
2483  cast<VTSDNode>(N->getOperand(1))->getVT()
2484  .getVectorElementType(),
2485  WidenVT.getVectorNumElements());
2486  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
2487  return DAG.getNode(N->getOpcode(), SDLoc(N),
2488  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
2489 }
2490 
2491 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
2492  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2493  return GetWidenedVector(WidenVec);
2494 }
2495 
2496 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
2497  SDValue InOp = N->getOperand(0);
2498  EVT InVT = InOp.getValueType();
2499  EVT VT = N->getValueType(0);
2500  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2501  SDLoc dl(N);
2502 
2503  switch (getTypeAction(InVT)) {
2505  break;
2507  // If the incoming type is a vector that is being promoted, then
2508  // we know that the elements are arranged differently and that we
2509  // must perform the conversion using a stack slot.
2510  if (InVT.isVector())
2511  break;
2512 
2513  // If the InOp is promoted to the same size, convert it. Otherwise,
2514  // fall out of the switch and widen the promoted input.
2515  InOp = GetPromotedInteger(InOp);
2516  InVT = InOp.getValueType();
2517  if (WidenVT.bitsEq(InVT))
2518  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2519  break;
2526  break;
2528  // If the InOp is widened to the same size, convert it. Otherwise, fall
2529  // out of the switch and widen the widened input.
2530  InOp = GetWidenedVector(InOp);
2531  InVT = InOp.getValueType();
2532  if (WidenVT.bitsEq(InVT))
2533  // The input widens to the same size. Convert to the widen value.
2534  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2535  break;
2536  }
2537 
2538  unsigned WidenSize = WidenVT.getSizeInBits();
2539  unsigned InSize = InVT.getSizeInBits();
2540  // x86mmx is not an acceptable vector element type, so don't try.
2541  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
2542  // Determine new input vector type. The new input vector type will use
2543  // the same element type (if its a vector) or use the input type as a
2544  // vector. It is the same size as the type to widen to.
2545  EVT NewInVT;
2546  unsigned NewNumElts = WidenSize / InSize;
2547  if (InVT.isVector()) {
2548  EVT InEltVT = InVT.getVectorElementType();
2549  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
2550  WidenSize / InEltVT.getSizeInBits());
2551  } else {
2552  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
2553  }
2554 
2555  if (TLI.isTypeLegal(NewInVT)) {
2556  // Because the result and the input are different vector types, widening
2557  // the result could create a legal type but widening the input might make
2558  // it an illegal type that might lead to repeatedly splitting the input
2559  // and then widening it. To avoid this, we widen the input only if
2560  // it results in a legal type.
2561  SmallVector<SDValue, 16> Ops(NewNumElts);
2562  SDValue UndefVal = DAG.getUNDEF(InVT);
2563  Ops[0] = InOp;
2564  for (unsigned i = 1; i < NewNumElts; ++i)
2565  Ops[i] = UndefVal;
2566 
2567  SDValue NewVec;
2568  if (InVT.isVector())
2569  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
2570  else
2571  NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops);
2572  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
2573  }
2574  }
2575 
2576  return CreateStackStoreLoad(InOp, WidenVT);
2577 }
2578 
2579 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
2580  SDLoc dl(N);
2581  // Build a vector with undefined for the new nodes.
2582  EVT VT = N->getValueType(0);
2583 
2584  // Integer BUILD_VECTOR operands may be larger than the node's vector element
2585  // type. The UNDEFs need to have the same type as the existing operands.
2586  EVT EltVT = N->getOperand(0).getValueType();
2587  unsigned NumElts = VT.getVectorNumElements();
2588 
2589  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2590  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2591 
2592  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
2593  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
2594  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
2595 
2596  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, NewOps);
2597 }
2598 
2599 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
2600  EVT InVT = N->getOperand(0).getValueType();
2601  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2602  SDLoc dl(N);
2603  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2604  unsigned NumInElts = InVT.getVectorNumElements();
2605  unsigned NumOperands = N->getNumOperands();
2606 
2607  bool InputWidened = false; // Indicates we need to widen the input.
2608  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
2609  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
2610  // Add undef vectors to widen to correct length.
2611  unsigned NumConcat = WidenVT.getVectorNumElements() /
2612  InVT.getVectorNumElements();
2613  SDValue UndefVal = DAG.getUNDEF(InVT);
2614  SmallVector<SDValue, 16> Ops(NumConcat);
2615  for (unsigned i=0; i < NumOperands; ++i)
2616  Ops[i] = N->getOperand(i);
2617  for (unsigned i = NumOperands; i != NumConcat; ++i)
2618  Ops[i] = UndefVal;
2619  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
2620  }
2621  } else {
2622  InputWidened = true;
2623  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
2624  // The inputs and the result are widen to the same value.
2625  unsigned i;
2626  for (i=1; i < NumOperands; ++i)
2627  if (!N->getOperand(i).isUndef())
2628  break;
2629 
2630  if (i == NumOperands)
2631  // Everything but the first operand is an UNDEF so just return the
2632  // widened first operand.
2633  return GetWidenedVector(N->getOperand(0));
2634 
2635  if (NumOperands == 2) {
2636  // Replace concat of two operands with a shuffle.
2637  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
2638  for (unsigned i = 0; i < NumInElts; ++i) {
2639  MaskOps[i] = i;
2640  MaskOps[i + NumInElts] = i + WidenNumElts;
2641  }
2642  return DAG.getVectorShuffle(WidenVT, dl,
2643  GetWidenedVector(N->getOperand(0)),
2644  GetWidenedVector(N->getOperand(1)),
2645  MaskOps);
2646  }
2647  }
2648  }
2649 
2650  // Fall back to use extracts and build vector.
2651  EVT EltVT = WidenVT.getVectorElementType();
2652  SmallVector<SDValue, 16> Ops(WidenNumElts);
2653  unsigned Idx = 0;
2654  for (unsigned i=0; i < NumOperands; ++i) {
2655  SDValue InOp = N->getOperand(i);
2656  if (InputWidened)
2657  InOp = GetWidenedVector(InOp);
2658  for (unsigned j=0; j < NumInElts; ++j)
2659  Ops[Idx++] = DAG.getNode(
2660  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2661  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2662  }
2663  SDValue UndefVal = DAG.getUNDEF(EltVT);
2664  for (; Idx < WidenNumElts; ++Idx)
2665  Ops[Idx] = UndefVal;
2666  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops);
2667 }
2668 
2669 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
2670  EVT VT = N->getValueType(0);
2671  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2672  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2673  SDValue InOp = N->getOperand(0);
2674  SDValue Idx = N->getOperand(1);
2675  SDLoc dl(N);
2676 
2677  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2678  InOp = GetWidenedVector(InOp);
2679 
2680  EVT InVT = InOp.getValueType();
2681 
2682  // Check if we can just return the input vector after widening.
2683  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2684  if (IdxVal == 0 && InVT == WidenVT)
2685  return InOp;
2686 
2687  // Check if we can extract from the vector.
2688  unsigned InNumElts = InVT.getVectorNumElements();
2689  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2690  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
2691 
2692  // We could try widening the input to the right length but for now, extract
2693  // the original elements, fill the rest with undefs and build a vector.
2694  SmallVector<SDValue, 16> Ops(WidenNumElts);
2695  EVT EltVT = VT.getVectorElementType();
2696  unsigned NumElts = VT.getVectorNumElements();
2697  unsigned i;
2698  for (i=0; i < NumElts; ++i)
2699  Ops[i] =
2700  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2701  DAG.getConstant(IdxVal + i, dl,
2702  TLI.getVectorIdxTy(DAG.getDataLayout())));
2703 
2704  SDValue UndefVal = DAG.getUNDEF(EltVT);
2705  for (; i < WidenNumElts; ++i)
2706  Ops[i] = UndefVal;
2707  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops);
2708 }
2709 
2710 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
2711  SDValue InOp = GetWidenedVector(N->getOperand(0));
2712  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
2713  InOp.getValueType(), InOp,
2714  N->getOperand(1), N->getOperand(2));
2715 }
2716 
2717 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
2718  LoadSDNode *LD = cast<LoadSDNode>(N);
2719  ISD::LoadExtType ExtType = LD->getExtensionType();
2720 
2721  SDValue Result;
2722  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
2723  if (ExtType != ISD::NON_EXTLOAD)
2724  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2725  else
2726  Result = GenWidenVectorLoads(LdChain, LD);
2727 
2728  // If we generate a single load, we can use that for the chain. Otherwise,
2729  // build a factor node to remember the multiple loads are independent and
2730  // chain to that.
2731  SDValue NewChain;
2732  if (LdChain.size() == 1)
2733  NewChain = LdChain[0];
2734  else
2735  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
2736 
2737  // Modified the chain - switch anything that used the old chain to use
2738  // the new one.
2739  ReplaceValueWith(SDValue(N, 1), NewChain);
2740 
2741  return Result;
2742 }
2743 
2744 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
2745 
2746  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
2747  SDValue Mask = N->getMask();
2748  EVT MaskVT = Mask.getValueType();
2749  SDValue Src0 = GetWidenedVector(N->getSrc0());
2750  ISD::LoadExtType ExtType = N->getExtensionType();
2751  SDLoc dl(N);
2752 
2753  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
2754  Mask = GetWidenedVector(Mask);
2755  else {
2756  EVT BoolVT = getSetCCResultType(WidenVT);
2757 
2758  // We can't use ModifyToType() because we should fill the mask with
2759  // zeroes
2760  unsigned WidenNumElts = BoolVT.getVectorNumElements();
2761  unsigned MaskNumElts = MaskVT.getVectorNumElements();
2762 
2763  unsigned NumConcat = WidenNumElts / MaskNumElts;
2764  SmallVector<SDValue, 16> Ops(NumConcat);
2765  SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
2766  Ops[0] = Mask;
2767  for (unsigned i = 1; i != NumConcat; ++i)
2768  Ops[i] = ZeroVal;
2769 
2770  Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
2771  }
2772 
2773  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
2774  Mask, Src0, N->getMemoryVT(),
2775  N->getMemOperand(), ExtType,
2776  N->isExpandingLoad());
2777  // Legalize the chain result - switch anything that used the old chain to
2778  // use the new one.
2779  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2780  return Res;
2781 }
2782 
2783 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
2784 
2785  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2786  SDValue Mask = N->getMask();
2787  SDValue Src0 = GetWidenedVector(N->getValue());
2788  unsigned NumElts = WideVT.getVectorNumElements();
2789  SDLoc dl(N);
2790 
2791  // The mask should be widened as well
2792  Mask = WidenTargetBoolean(Mask, WideVT, true);
2793 
2794  // Widen the Index operand
2795  SDValue Index = N->getIndex();
2796  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
2797  Index.getValueType().getScalarType(),
2798  NumElts);
2799  Index = ModifyToType(Index, WideIndexVT);
2800  SDValue Ops[] = { N->getChain(), Src0, Mask, N->getBasePtr(), Index };
2801  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
2802  N->getMemoryVT(), dl, Ops,
2803  N->getMemOperand());
2804 
2805  // Legalize the chain result - switch anything that used the old chain to
2806  // use the new one.
2807  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2808  return Res;
2809 }
2810 
2811 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
2812  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2813  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
2814  WidenVT, N->getOperand(0));
2815 }
2816 
2817 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
2818  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2819  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2820 
2821  SDValue Cond1 = N->getOperand(0);
2822  EVT CondVT = Cond1.getValueType();
2823  if (CondVT.isVector()) {
2824  EVT CondEltVT = CondVT.getVectorElementType();
2825  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
2826  CondEltVT, WidenNumElts);
2827  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
2828  Cond1 = GetWidenedVector(Cond1);
2829 
2830  // If we have to split the condition there is no point in widening the
2831  // select. This would result in an cycle of widening the select ->
2832  // widening the condition operand -> splitting the condition operand ->
2833  // splitting the select -> widening the select. Instead split this select
2834  // further and widen the resulting type.
2835  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
2836  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
2837  SDValue Res = ModifyToType(SplitSelect, WidenVT);
2838  return Res;
2839  }
2840 
2841  if (Cond1.getValueType() != CondWidenVT)
2842  Cond1 = ModifyToType(Cond1, CondWidenVT);
2843  }
2844 
2845  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
2846  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
2847  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
2848  return DAG.getNode(N->getOpcode(), SDLoc(N),
2849  WidenVT, Cond1, InOp1, InOp2);
2850 }
2851 
2852 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
2853  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
2854  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
2855  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2856  InOp1.getValueType(), N->getOperand(0),
2857  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
2858 }
2859 
2860 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
2861  assert(N->getValueType(0).isVector() ==
2862  N->getOperand(0).getValueType().isVector() &&
2863  "Scalar/Vector type mismatch");
2864  if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N);
2865 
2866  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2867  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2868  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2869  return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT,
2870  InOp1, InOp2, N->getOperand(2));
2871 }
2872 
2873 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
2874  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2875  return DAG.getUNDEF(WidenVT);
2876 }
2877 
2878 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
2879  EVT VT = N->getValueType(0);
2880  SDLoc dl(N);
2881 
2882  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2883  unsigned NumElts = VT.getVectorNumElements();
2884  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2885 
2886  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2887  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2888 
2889  // Adjust mask based on new input vector length.
2890  SmallVector<int, 16> NewMask;
2891  for (unsigned i = 0; i != NumElts; ++i) {
2892  int Idx = N->getMaskElt(i);
2893  if (Idx < (int)NumElts)
2894  NewMask.push_back(Idx);
2895  else
2896  NewMask.push_back(Idx - NumElts + WidenNumElts);
2897  }
2898  for (unsigned i = NumElts; i != WidenNumElts; ++i)
2899  NewMask.push_back(-1);
2900  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
2901 }
2902 
2903 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
2904  assert(N->getValueType(0).isVector() &&
2905  N->getOperand(0).getValueType().isVector() &&
2906  "Operands must be vectors");
2907  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2908  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2909 
2910  SDValue InOp1 = N->getOperand(0);
2911  EVT InVT = InOp1.getValueType();
2912  assert(InVT.isVector() && "can not widen non-vector type");
2913  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
2914  InVT.getVectorElementType(), WidenNumElts);
2915 
2916  // The input and output types often differ here, and it could be that while
2917  // we'd prefer to widen the result type, the input operands have been split.
2918  // In this case, we also need to split the result of this node as well.
2919  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
2920  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
2921  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
2922  return Res;
2923  }
2924 
2925  InOp1 = GetWidenedVector(InOp1);
2926  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2927 
2928  // Assume that the input and output will be widen appropriately. If not,
2929  // we will have to unroll it at some point.
2930  assert(InOp1.getValueType() == WidenInVT &&
2931  InOp2.getValueType() == WidenInVT &&
2932  "Input not widened to expected type!");
2933  (void)WidenInVT;
2934  return DAG.getNode(ISD::SETCC, SDLoc(N),
2935  WidenVT, InOp1, InOp2, N->getOperand(2));
2936 }
2937 
2938 
2939 //===----------------------------------------------------------------------===//
2940 // Widen Vector Operand
2941 //===----------------------------------------------------------------------===//
2942 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
2943  DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
2944  N->dump(&DAG);
2945  dbgs() << "\n");
2946  SDValue Res = SDValue();
2947 
2948  // See if the target wants to custom widen this node.
2949  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2950  return false;
2951 
2952  switch (N->getOpcode()) {
2953  default:
2954 #ifndef NDEBUG
2955  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
2956  N->dump(&DAG);
2957  dbgs() << "\n";
2958 #endif
2959  llvm_unreachable("Do not know how to widen this operator's operand!");
2960 
2961  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
2962  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
2963  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
2964  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
2965  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
2966  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
2967  case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
2968  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
2969  case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
2970 
2971  case ISD::ANY_EXTEND:
2972  case ISD::SIGN_EXTEND:
2973  case ISD::ZERO_EXTEND:
2974  Res = WidenVecOp_EXTEND(N);
2975  break;
2976 
2977  case ISD::FP_EXTEND:
2978  case ISD::FP_TO_SINT:
2979  case ISD::FP_TO_UINT:
2980  case ISD::SINT_TO_FP:
2981  case ISD::UINT_TO_FP:
2982  case ISD::TRUNCATE:
2983  Res = WidenVecOp_Convert(N);
2984  break;
2985  }
2986 
2987  // If Res is null, the sub-method took care of registering the result.
2988  if (!Res.getNode()) return false;
2989 
2990  // If the result is N, the sub-method updated N in place. Tell the legalizer
2991  // core about this.
2992  if (Res.getNode() == N)
2993  return true;
2994 
2995 
2996  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2997  "Invalid operand expansion");
2998 
2999  ReplaceValueWith(SDValue(N, 0), Res);
3000  return false;
3001 }
3002 
3003 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
3004  SDLoc DL(N);
3005  EVT VT = N->getValueType(0);
3006 
3007  SDValue InOp = N->getOperand(0);
3008  // If some legalization strategy other than widening is used on the operand,
3009  // we can't safely assume that just extending the low lanes is the correct
3010  // transformation.
3011  if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector)
3012  return WidenVecOp_Convert(N);
3013  InOp = GetWidenedVector(InOp);
3016  "Input wasn't widened!");
3017 
3018  // We may need to further widen the operand until it has the same total
3019  // vector size as the result.
3020  EVT InVT = InOp.getValueType();
3021  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
3022  EVT InEltVT = InVT.getVectorElementType();
3023  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
3024  EVT FixedVT = (MVT::SimpleValueType)i;
3025  EVT FixedEltVT = FixedVT.getVectorElementType();
3026  if (TLI.isTypeLegal(FixedVT) &&
3027  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
3028  FixedEltVT == InEltVT) {
3029  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
3030  "Not enough elements in the fixed type for the operand!");
3031  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
3032  "We can't have the same type as we started with!");
3033  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
3034  InOp = DAG.getNode(
3035  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
3036  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3037  else
3038  InOp = DAG.getNode(
3039  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
3040  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3041  break;
3042  }
3043  }
3044  InVT = InOp.getValueType();
3045  if (InVT.getSizeInBits() != VT.getSizeInBits())
3046  // We couldn't find a legal vector type that was a widening of the input
3047  // and could be extended in-register to the result type, so we have to
3048  // scalarize.
3049  return WidenVecOp_Convert(N);
3050  }
3051 
3052  // Use special DAG nodes to represent the operation of extending the
3053  // low lanes.
3054  switch (N->getOpcode()) {
3055  default:
3056  llvm_unreachable("Extend legalization on on extend operation!");
3057  case ISD::ANY_EXTEND:
3058  return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
3059  case ISD::SIGN_EXTEND:
3060  return DAG.getSignExtendVectorInReg(InOp, DL, VT);
3061  case ISD::ZERO_EXTEND:
3062  return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
3063  }
3064 }
3065 
3066 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
3067  // The result (and first input) is legal, but the second input is illegal.
3068  // We can't do much to fix that, so just unroll and let the extracts off of
3069  // the second input be widened as needed later.
3070  return DAG.UnrollVectorOp(N);
3071 }
3072 
3073 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
3074  // Since the result is legal and the input is illegal, it is unlikely that we
3075  // can fix the input to a legal type so unroll the convert into some scalar
3076  // code and create a nasty build vector.
3077  EVT VT = N->getValueType(0);
3078  EVT EltVT = VT.getVectorElementType();
3079  SDLoc dl(N);
3080  unsigned NumElts = VT.getVectorNumElements();
3081  SDValue InOp = N->getOperand(0);
3082  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3083  InOp = GetWidenedVector(InOp);
3084  EVT InVT = InOp.getValueType();
3085  EVT InEltVT = InVT.getVectorElementType();
3086 
3087  unsigned Opcode = N->getOpcode();
3088  SmallVector<SDValue, 16> Ops(NumElts);
3089  for (unsigned i=0; i < NumElts; ++i)
3090  Ops[i] = DAG.getNode(
3091  Opcode, dl, EltVT,
3092  DAG.getNode(
3093  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
3094  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3095 
3096  return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
3097 }
3098 
3099 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
3100  EVT VT = N->getValueType(0);
3101  SDValue InOp = GetWidenedVector(N->getOperand(0));
3102  EVT InWidenVT = InOp.getValueType();
3103  SDLoc dl(N);
3104 
3105  // Check if we can convert between two legal vector types and extract.
3106  unsigned InWidenSize = InWidenVT.getSizeInBits();
3107  unsigned Size = VT.getSizeInBits();
3108  // x86mmx is not an acceptable vector element type, so don't try.
3109  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
3110  unsigned NewNumElts = InWidenSize / Size;
3111  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
3112  if (TLI.isTypeLegal(NewVT)) {
3113  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
3114  return DAG.getNode(
3115  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
3116  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3117  }
3118  }
3119 
3120  return CreateStackStoreLoad(InOp, VT);
3121 }
3122 
3123 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
3124  // If the input vector is not legal, it is likely that we will not find a
3125  // legal vector of the same size. Replace the concatenate vector with a
3126  // nasty build vector.
3127  EVT VT = N->getValueType(0);
3128  EVT EltVT = VT.getVectorElementType();
3129  SDLoc dl(N);
3130  unsigned NumElts = VT.getVectorNumElements();
3131  SmallVector<SDValue, 16> Ops(NumElts);
3132 
3133  EVT InVT = N->getOperand(0).getValueType();
3134  unsigned NumInElts = InVT.getVectorNumElements();
3135 
3136  unsigned Idx = 0;
3137  unsigned NumOperands = N->getNumOperands();
3138  for (unsigned i=0; i < NumOperands; ++i) {
3139  SDValue InOp = N->getOperand(i);
3140  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3141  InOp = GetWidenedVector(InOp);
3142  for (unsigned j=0; j < NumInElts; ++j)
3143  Ops[Idx++] = DAG.getNode(
3144  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3145  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3146  }
3147  return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
3148 }
3149 
3150 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3151  SDValue InOp = GetWidenedVector(N->getOperand(0));
3152  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
3153  N->getValueType(0), InOp, N->getOperand(1));
3154 }
3155 
3156 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3157  SDValue InOp = GetWidenedVector(N->getOperand(0));
3158  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3159  N->getValueType(0), InOp, N->getOperand(1));
3160 }
3161 
3162 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
3163  // We have to widen the value, but we want only to store the original
3164  // vector type.
3165  StoreSDNode *ST = cast<StoreSDNode>(N);
3166 
3167  SmallVector<SDValue, 16> StChain;
3168  if (ST->isTruncatingStore())
3169  GenWidenVectorTruncStores(StChain, ST);
3170  else
3171  GenWidenVectorStores(StChain, ST);
3172 
3173  if (StChain.size() == 1)
3174  return StChain[0];
3175  else
3176  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
3177 }
3178 
3179 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
3180  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
3181  SDValue Mask = MST->getMask();
3182  EVT MaskVT = Mask.getValueType();
3183  SDValue StVal = MST->getValue();
3184  // Widen the value
3185  SDValue WideVal = GetWidenedVector(StVal);
3186  SDLoc dl(N);
3187 
3188  if (OpNo == 2 || getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3189  Mask = GetWidenedVector(Mask);
3190  else {
3191  // The mask should be widened as well.
3192  EVT BoolVT = getSetCCResultType(WideVal.getValueType());
3193  // We can't use ModifyToType() because we should fill the mask with
3194  // zeroes.
3195  unsigned WidenNumElts = BoolVT.getVectorNumElements();
3196  unsigned MaskNumElts = MaskVT.getVectorNumElements();
3197 
3198  unsigned NumConcat = WidenNumElts / MaskNumElts;
3199  SmallVector<SDValue, 16> Ops(NumConcat);
3200  SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
3201  Ops[0] = Mask;
3202  for (unsigned i = 1; i != NumConcat; ++i)
3203  Ops[i] = ZeroVal;
3204 
3205  Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
3206  }
3207  assert(Mask.getValueType().getVectorNumElements() ==
3208  WideVal.getValueType().getVectorNumElements() &&
3209  "Mask and data vectors should have the same number of elements");
3210  return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
3211  Mask, MST->getMemoryVT(), MST->getMemOperand(),
3212  false, MST->isCompressingStore());
3213 }
3214 
3215 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
3216  assert(OpNo == 1 && "Can widen only data operand of mscatter");
3217  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
3218  SDValue DataOp = MSC->getValue();
3219  SDValue Mask = MSC->getMask();
3220 
3221  // Widen the value.
3222  SDValue WideVal = GetWidenedVector(DataOp);
3223  EVT WideVT = WideVal.getValueType();
3224  unsigned NumElts = WideVal.getValueType().getVectorNumElements();
3225  SDLoc dl(N);
3226 
3227  // The mask should be widened as well.
3228  Mask = WidenTargetBoolean(Mask, WideVT, true);
3229 
3230  // Widen index.
3231  SDValue Index = MSC->getIndex();
3232  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3233  Index.getValueType().getScalarType(),
3234  NumElts);
3235  Index = ModifyToType(Index, WideIndexVT);
3236 
3237  SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index};
3238  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
3239  MSC->getMemoryVT(), dl, Ops,
3240  MSC->getMemOperand());
3241 }
3242 
3243 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
3244  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
3245  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3246  SDLoc dl(N);
3247 
3248  // WARNING: In this code we widen the compare instruction with garbage.
3249  // This garbage may contain denormal floats which may be slow. Is this a real
3250  // concern ? Should we zero the unused lanes if this is a float compare ?
3251 
3252  // Get a new SETCC node to compare the newly widened operands.
3253  // Only some of the compared elements are legal.
3254  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3255  InOp0.getValueType());
3256  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
3257  SVT, InOp0, InOp1, N->getOperand(2));
3258 
3259  // Extract the needed results from the result vector.
3260  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
3261  SVT.getVectorElementType(),
3263  SDValue CC = DAG.getNode(
3264  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
3265  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3266 
3267  return PromoteTargetBoolean(CC, N->getValueType(0));
3268 }
3269 
3270 
3271 //===----------------------------------------------------------------------===//
3272 // Vector Widening Utilities
3273 //===----------------------------------------------------------------------===//
3274 
3275 // Utility function to find the type to chop up a widen vector for load/store
3276 // TLI: Target lowering used to determine legal types.
3277 // Width: Width left need to load/store.
3278 // WidenVT: The widen vector type to load to/store from
3279 // Align: If 0, don't allow use of a wider type
3280 // WidenEx: If Align is not 0, the amount additional we can load/store from.
3281 
3282 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
3283  unsigned Width, EVT WidenVT,
3284  unsigned Align = 0, unsigned WidenEx = 0) {
3285  EVT WidenEltVT = WidenVT.getVectorElementType();
3286  unsigned WidenWidth = WidenVT.getSizeInBits();
3287  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
3288  unsigned AlignInBits = Align*8;
3289 
3290  // If we have one element to load/store, return it.
3291  EVT RetVT = WidenEltVT;
3292  if (Width == WidenEltWidth)
3293  return RetVT;
3294 
3295  // See if there is larger legal integer than the element type to load/store.
3296  unsigned VT;
3297  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
3298  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
3299  EVT MemVT((MVT::SimpleValueType) VT);
3300  unsigned MemVTWidth = MemVT.getSizeInBits();
3301  if (MemVT.getSizeInBits() <= WidenEltWidth)
3302  break;
3303  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
3304  if ((Action == TargetLowering::TypeLegal ||
3306  (WidenWidth % MemVTWidth) == 0 &&
3307  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3308  (MemVTWidth <= Width ||
3309  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3310  RetVT = MemVT;
3311  break;
3312  }
3313  }
3314 
3315  // See if there is a larger vector type to load/store that has the same vector
3316  // element type and is evenly divisible with the WidenVT.
3317  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
3318  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
3319  EVT MemVT = (MVT::SimpleValueType) VT;
3320  unsigned MemVTWidth = MemVT.getSizeInBits();
3321  if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
3322  (WidenWidth % MemVTWidth) == 0 &&
3323  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3324  (MemVTWidth <= Width ||
3325  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3326  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
3327  return MemVT;
3328  }
3329  }
3330 
3331  return RetVT;
3332 }
3333 
3334 // Builds a vector type from scalar loads
3335 // VecTy: Resulting Vector type
3336 // LDOps: Load operators to build a vector type
3337 // [Start,End) the list of loads to use.
3339  SmallVectorImpl<SDValue> &LdOps,
3340  unsigned Start, unsigned End) {
3341  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3342  SDLoc dl(LdOps[Start]);
3343  EVT LdTy = LdOps[Start].getValueType();
3344  unsigned Width = VecTy.getSizeInBits();
3345  unsigned NumElts = Width / LdTy.getSizeInBits();
3346  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
3347 
3348  unsigned Idx = 1;
3349  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
3350 
3351  for (unsigned i = Start + 1; i != End; ++i) {
3352  EVT NewLdTy = LdOps[i].getValueType();
3353  if (NewLdTy != LdTy) {
3354  NumElts = Width / NewLdTy.getSizeInBits();
3355  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
3356  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
3357  // Readjust position and vector position based on new load type.
3358  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
3359  LdTy = NewLdTy;
3360  }
3361  VecOp = DAG.getNode(
3362  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
3363  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3364  }
3365  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
3366 }
3367 
3368 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
3369  LoadSDNode *LD) {
3370  // The strategy assumes that we can efficiently load power-of-two widths.
3371  // The routine chops the vector into the largest vector loads with the same
3372  // element type or scalar loads and then recombines it to the widen vector
3373  // type.
3374  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3375  unsigned WidenWidth = WidenVT.getSizeInBits();
3376  EVT LdVT = LD->getMemoryVT();
3377  SDLoc dl(LD);
3378  assert(LdVT.isVector() && WidenVT.isVector());
3379  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
3380 
3381  // Load information
3382  SDValue Chain = LD->getChain();
3383  SDValue BasePtr = LD->getBasePtr();
3384  unsigned Align = LD->getAlignment();
3385  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3386  AAMDNodes AAInfo = LD->getAAInfo();
3387 
3388  int LdWidth = LdVT.getSizeInBits();
3389  int WidthDiff = WidenWidth - LdWidth;
3390  unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
3391 
3392  // Find the vector type that can load from.
3393  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3394  int NewVTWidth = NewVT.getSizeInBits();
3395  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
3396  Align, MMOFlags, AAInfo);
3397  LdChain.push_back(LdOp.getValue(1));
3398 
3399  // Check if we can load the element with one instruction.
3400  if (LdWidth <= NewVTWidth) {
3401  if (!NewVT.isVector()) {
3402  unsigned NumElts = WidenWidth / NewVTWidth;
3403  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
3404  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
3405  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
3406  }
3407  if (NewVT == WidenVT)
3408  return LdOp;
3409 
3410  assert(WidenWidth % NewVTWidth == 0);
3411  unsigned NumConcat = WidenWidth / NewVTWidth;
3412  SmallVector<SDValue, 16> ConcatOps(NumConcat);
3413  SDValue UndefVal = DAG.getUNDEF(NewVT);
3414  ConcatOps[0] = LdOp;
3415  for (unsigned i = 1; i != NumConcat; ++i)
3416  ConcatOps[i] = UndefVal;
3417  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
3418  }
3419 
3420  // Load vector by using multiple loads from largest vector to scalar.
3422  LdOps.push_back(LdOp);
3423 
3424  LdWidth -= NewVTWidth;
3425  unsigned Offset = 0;
3426 
3427  while (LdWidth > 0) {
3428  unsigned Increment = NewVTWidth / 8;
3429  Offset += Increment;
3430  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3431  DAG.getConstant(Increment, dl, BasePtr.getValueType()));
3432 
3433  SDValue L;
3434  if (LdWidth < NewVTWidth) {
3435  // The current type we are using is too large. Find a better size.
3436  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3437  NewVTWidth = NewVT.getSizeInBits();
3438  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3439  LD->getPointerInfo().getWithOffset(Offset),
3440  MinAlign(Align, Increment), MMOFlags, AAInfo);
3441  LdChain.push_back(L.getValue(1));
3442  if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
3443  // Later code assumes the vector loads produced will be mergeable, so we
3444  // must pad the final entry up to the previous width. Scalars are
3445  // combined separately.
3447  Loads.push_back(L);
3448  unsigned size = L->getValueSizeInBits(0);
3449  while (size < LdOp->getValueSizeInBits(0)) {
3450  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
3451  size += L->getValueSizeInBits(0);
3452  }
3453  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
3454  }
3455  } else {
3456  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3457  LD->getPointerInfo().getWithOffset(Offset),
3458  MinAlign(Align, Increment), MMOFlags, AAInfo);
3459  LdChain.push_back(L.getValue(1));
3460  }
3461 
3462  LdOps.push_back(L);
3463 
3464 
3465  LdWidth -= NewVTWidth;
3466  }
3467 
3468  // Build the vector from the load operations.
3469  unsigned End = LdOps.size();
3470  if (!LdOps[0].getValueType().isVector())
3471  // All the loads are scalar loads.
3472  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
3473 
3474  // If the load contains vectors, build the vector using concat vector.
3475  // All of the vectors used to load are power-of-2, and the scalar loads can be
3476  // combined to make a power-of-2 vector.
3477  SmallVector<SDValue, 16> ConcatOps(End);
3478  int i = End - 1;
3479  int Idx = End;
3480  EVT LdTy = LdOps[i].getValueType();
3481  // First, combine the scalar loads to a vector.
3482  if (!LdTy.isVector()) {
3483  for (--i; i >= 0; --i) {
3484  LdTy = LdOps[i].getValueType();
3485  if (LdTy.isVector())
3486  break;
3487  }
3488  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
3489  }
3490  ConcatOps[--Idx] = LdOps[i];
3491  for (--i; i >= 0; --i) {
3492  EVT NewLdTy = LdOps[i].getValueType();
3493  if (NewLdTy != LdTy) {
3494  // Create a larger vector.
3495  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
3496  makeArrayRef(&ConcatOps[Idx], End - Idx));
3497  Idx = End - 1;
3498  LdTy = NewLdTy;
3499  }
3500  ConcatOps[--Idx] = LdOps[i];
3501  }
3502 
3503  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
3504  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3505  makeArrayRef(&ConcatOps[Idx], End - Idx));
3506 
3507  // We need to fill the rest with undefs to build the vector.
3508  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
3509  SmallVector<SDValue, 16> WidenOps(NumOps);
3510  SDValue UndefVal = DAG.getUNDEF(LdTy);
3511  {
3512  unsigned i = 0;
3513  for (; i != End-Idx; ++i)
3514  WidenOps[i] = ConcatOps[Idx+i];
3515  for (; i != NumOps; ++i)
3516  WidenOps[i] = UndefVal;
3517  }
3518  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
3519 }
3520 
3521 SDValue
3522 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
3523  LoadSDNode *LD,
3524  ISD::LoadExtType ExtType) {
3525  // For extension loads, it may not be more efficient to chop up the vector
3526  // and then extend it. Instead, we unroll the load and build a new vector.
3527  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3528  EVT LdVT = LD->getMemoryVT();
3529  SDLoc dl(LD);
3530  assert(LdVT.isVector() && WidenVT.isVector());
3531 
3532  // Load information
3533  SDValue Chain = LD->getChain();
3534  SDValue BasePtr = LD->getBasePtr();
3535  unsigned Align = LD->getAlignment();
3536  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3537  AAMDNodes AAInfo = LD->getAAInfo();
3538 
3539  EVT EltVT = WidenVT.getVectorElementType();
3540  EVT LdEltVT = LdVT.getVectorElementType();
3541  unsigned NumElts = LdVT.getVectorNumElements();
3542 
3543  // Load each element and widen.
3544  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3545  SmallVector<SDValue, 16> Ops(WidenNumElts);
3546  unsigned Increment = LdEltVT.getSizeInBits() / 8;
3547  Ops[0] =
3548  DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
3549  LdEltVT, Align, MMOFlags, AAInfo);
3550  LdChain.push_back(Ops[0].getValue(1));
3551  unsigned i = 0, Offset = Increment;
3552  for (i=1; i < NumElts; ++i, Offset += Increment) {
3553  SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
3554  BasePtr,
3555  DAG.getConstant(Offset, dl,
3556  BasePtr.getValueType()));
3557  Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
3558  LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
3559  Align, MMOFlags, AAInfo);
3560  LdChain.push_back(Ops[i].getValue(1));
3561  }
3562 
3563  // Fill the rest with undefs.
3564  SDValue UndefVal = DAG.getUNDEF(EltVT);
3565  for (; i != WidenNumElts; ++i)
3566  Ops[i] = UndefVal;
3567 
3568  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops);
3569 }
3570 
3571 
3572 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
3573  StoreSDNode *ST) {
3574  // The strategy assumes that we can efficiently store power-of-two widths.
3575  // The routine chops the vector into the largest vector stores with the same
3576  // element type or scalar stores.
3577  SDValue Chain = ST->getChain();
3578  SDValue BasePtr = ST->getBasePtr();
3579  unsigned Align = ST->getAlignment();
3580  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
3581  AAMDNodes AAInfo = ST->getAAInfo();
3582  SDValue ValOp = GetWidenedVector(ST->getValue());
3583  SDLoc dl(ST);
3584 
3585  EVT StVT = ST->getMemoryVT();
3586  unsigned StWidth = StVT.getSizeInBits();
3587  EVT ValVT = ValOp.getValueType();
3588  unsigned ValWidth = ValVT.getSizeInBits();
3589  EVT ValEltVT = ValVT.getVectorElementType();
3590  unsigned ValEltWidth = ValEltVT.getSizeInBits();
3591  assert(StVT.getVectorElementType() == ValEltVT);
3592 
3593  int Idx = 0; // current index to store
3594  unsigned Offset = 0; // offset from base to store
3595  while (StWidth != 0) {
3596  // Find the largest vector type we can store with.
3597  EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
3598  unsigned NewVTWidth = NewVT.getSizeInBits();
3599  unsigned Increment = NewVTWidth / 8;
3600  if (NewVT.isVector()) {
3601  unsigned NumVTElts = NewVT.getVectorNumElements();
3602  do {
3603  SDValue EOp = DAG.getNode(
3604  ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
3605  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3606  StChain.push_back(DAG.getStore(
3607  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
3608  MinAlign(Align, Offset), MMOFlags, AAInfo));
3609  StWidth -= NewVTWidth;
3610  Offset += Increment;
3611  Idx += NumVTElts;
3612  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3613  DAG.getConstant(Increment, dl,
3614  BasePtr.getValueType()));
3615  } while (StWidth != 0 && StWidth >= NewVTWidth);
3616  } else {
3617  // Cast the vector to the scalar type we can store.
3618  unsigned NumElts = ValWidth / NewVTWidth;
3619  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
3620  SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
3621  // Readjust index position based on new vector type.
3622  Idx = Idx * ValEltWidth / NewVTWidth;
3623  do {
3624  SDValue EOp = DAG.getNode(
3625  ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
3626  DAG.getConstant(Idx++, dl,
3627  TLI.getVectorIdxTy(DAG.getDataLayout())));
3628  StChain.push_back(DAG.getStore(
3629  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
3630  MinAlign(Align, Offset), MMOFlags, AAInfo));
3631  StWidth -= NewVTWidth;
3632  Offset += Increment;
3633  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3634  DAG.getConstant(Increment, dl,
3635  BasePtr.getValueType()));
3636  } while (StWidth != 0 && StWidth >= NewVTWidth);
3637  // Restore index back to be relative to the original widen element type.
3638  Idx = Idx * NewVTWidth / ValEltWidth;
3639  }
3640  }
3641 }
3642 
3643 void
3644 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
3645  StoreSDNode *ST) {
3646  // For extension loads, it may not be more efficient to truncate the vector
3647  // and then store it. Instead, we extract each element and then store it.
3648  SDValue Chain = ST->getChain();
3649  SDValue BasePtr = ST->getBasePtr();
3650  unsigned Align = ST->getAlignment();
3651  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
3652  AAMDNodes AAInfo = ST->getAAInfo();
3653  SDValue ValOp = GetWidenedVector(ST->getValue());
3654  SDLoc dl(ST);
3655 
3656  EVT StVT = ST->getMemoryVT();
3657  EVT ValVT = ValOp.getValueType();
3658 
3659  // It must be true that the wide vector type is bigger than where we need to
3660  // store.
3661  assert(StVT.isVector() && ValOp.getValueType().isVector());
3662  assert(StVT.bitsLT(ValOp.getValueType()));
3663 
3664  // For truncating stores, we can not play the tricks of chopping legal vector
3665  // types and bitcast it to the right type. Instead, we unroll the store.
3666  EVT StEltVT = StVT.getVectorElementType();
3667  EVT ValEltVT = ValVT.getVectorElementType();
3668  unsigned Increment = ValEltVT.getSizeInBits() / 8;
3669  unsigned NumElts = StVT.getVectorNumElements();
3670  SDValue EOp = DAG.getNode(
3671  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
3672  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3673  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
3674  ST->getPointerInfo(), StEltVT, Align,
3675  MMOFlags, AAInfo));
3676  unsigned Offset = Increment;
3677  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
3678  SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
3679  BasePtr,
3680  DAG.getConstant(Offset, dl,
3681  BasePtr.getValueType()));
3682  SDValue EOp = DAG.getNode(
3683  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
3684  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3685  StChain.push_back(DAG.getTruncStore(
3686  Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
3687  StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
3688  }
3689 }
3690 
3691 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
3692 /// input vector must have the same element type as NVT.
3693 /// FillWithZeroes specifies that the vector should be widened with zeroes.
3694 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
3695  bool FillWithZeroes) {
3696  // Note that InOp might have been widened so it might already have
3697  // the right width or it might need be narrowed.
3698  EVT InVT = InOp.getValueType();
3700  "input and widen element type must match");
3701  SDLoc dl(InOp);
3702 
3703  // Check if InOp already has the right width.
3704  if (InVT == NVT)
3705  return InOp;
3706 
3707  unsigned InNumElts = InVT.getVectorNumElements();
3708  unsigned WidenNumElts = NVT.getVectorNumElements();
3709  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
3710  unsigned NumConcat = WidenNumElts / InNumElts;
3711  SmallVector<SDValue, 16> Ops(NumConcat);
3712  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
3713  DAG.getUNDEF(InVT);
3714  Ops[0] = InOp;
3715  for (unsigned i = 1; i != NumConcat; ++i)
3716  Ops[i] = FillVal;
3717 
3718  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
3719  }
3720 
3721  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
3722  return DAG.getNode(
3723  ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
3724  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3725 
3726  // Fall back to extract and build.
3727  SmallVector<SDValue, 16> Ops(WidenNumElts);
3728  EVT EltVT = NVT.getVectorElementType();
3729  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
3730  unsigned Idx;
3731  for (Idx = 0; Idx < MinNumElts; ++Idx)
3732  Ops[Idx] = DAG.getNode(
3733  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3734  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3735 
3736  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
3737  DAG.getUNDEF(EltVT);
3738  for ( ; Idx < WidenNumElts; ++Idx)
3739  Ops[Idx] = FillVal;
3740  return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Ops);
3741 }
MachineLoop * L
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:500
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:467
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:524
SDValue getValue(unsigned R) const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
Flags getFlags() const
Return the raw flags of the source value,.
LLVMContext * getContext() const
Definition: SelectionDAG.h:333
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:304
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
size_t i
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:313
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:449
SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:699
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getSrc0() const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:817
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:440
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:330
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:299
uint64_t High
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:369
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:212
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:133
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
A description of a memory reference used in the backend.
struct fuzzer::@269 Flags
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:440
Shift and rotation operations.
Definition: ISDOpcodes.h:344
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:327
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:233
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored...
int getMaskElt(unsigned Idx) const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
ISD::LoadExtType getExtensionType() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:123
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:239
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:410
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:363
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:737
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
EVT getMemoryVT() const
Return the type of the in-memory value.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:487
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:427
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Idx) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:328
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:453
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:274
SDNode * getNode() const
get the SDNode which holds the desired result
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:589
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:262
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:268
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:399
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
Simple binary floating point operators.
Definition: ISDOpcodes.h:246
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:818
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:279
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
uint32_t Offset
static EVT getFloatingPointVT(unsigned BitWidth)
getFloatingPointVT - Returns the EVT that represents a floating point type with the given number of b...
Definition: ValueTypes.h:55
static const unsigned End
const SDValue & getBasePtr() const
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
bool isVolatile() const
const SDValue & getValue() const
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:350
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:485
EVT - Extended Value Type.
Definition: ValueTypes.h:31
const SDValue & getMask() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:194
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
This class contains a discriminated union of information about pointers in memory operands...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const MachinePointerInfo & getPointerInfo() const
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
The memory access writes data.
SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:285
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:166
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:482
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isVector(MCInstrInfo const &MCII, MCInst const &MCI)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:649
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:625
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:347
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
const SDValue & getValue() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:354
op_iterator op_begin() const
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:400
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:403
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:259
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
Definition: ISDOpcodes.h:527
iterator_range< value_op_iterator > op_values() const
Flags
Flags values. These may be or'd together.
const SDValue & getMask() const
The memory access reads data.
These are IR-level optimization flags that may be propagated to SDNodes.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:333
bool isUndef() const
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:418
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:536
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
#define N
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
op_iterator op_end() const
This class is used to represent an MSCATTER node.
const SDValue & getIndex() const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:291
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:118
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:249
const SDValue & getBasePtr() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:331
#define DEBUG(X)
Definition: Debug.h:100
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:377
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:197
Conversion operators.
Definition: ISDOpcodes.h:397
BooleanContent
Enum that describes how the target represents true/false values.
int * Ptr
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:381
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:406
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool isBigEndian() const
Definition: DataLayout.h:221
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:516
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:321
const SDNodeFlags * getFlags() const
This could be defined as a virtual function and implemented more simply and directly, but it is not to avoid creating a vtable for this class.
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:326
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:248
This class is used to represent ISD::LOAD nodes.