LLVM  7.0.0svn
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16 // types.
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "legalize-types"
30 
31 //===----------------------------------------------------------------------===//
32 // Result Vector Scalarization: <1 x ty> -> ty.
33 //===----------------------------------------------------------------------===//
34 
35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
37  dbgs() << "\n");
38  SDValue R = SDValue();
39 
40  switch (N->getOpcode()) {
41  default:
42 #ifndef NDEBUG
43  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
44  N->dump(&DAG);
45  dbgs() << "\n";
46 #endif
47  report_fatal_error("Do not know how to scalarize the result of this "
48  "operator!\n");
49 
50  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
51  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
52  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
53  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
54  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
55  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
56  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
70  R = ScalarizeVecRes_VecInregOp(N);
71  break;
72  case ISD::ANY_EXTEND:
73  case ISD::BITREVERSE:
74  case ISD::BSWAP:
75  case ISD::CTLZ:
77  case ISD::CTPOP:
78  case ISD::CTTZ:
80  case ISD::FABS:
81  case ISD::FCEIL:
82  case ISD::FCOS:
83  case ISD::FEXP:
84  case ISD::FEXP2:
85  case ISD::FFLOOR:
86  case ISD::FLOG:
87  case ISD::FLOG10:
88  case ISD::FLOG2:
89  case ISD::FNEARBYINT:
90  case ISD::FNEG:
91  case ISD::FP_EXTEND:
92  case ISD::FP_TO_SINT:
93  case ISD::FP_TO_UINT:
94  case ISD::FRINT:
95  case ISD::FROUND:
96  case ISD::FSIN:
97  case ISD::FSQRT:
98  case ISD::FTRUNC:
99  case ISD::SIGN_EXTEND:
100  case ISD::SINT_TO_FP:
101  case ISD::TRUNCATE:
102  case ISD::UINT_TO_FP:
103  case ISD::ZERO_EXTEND:
104  case ISD::FCANONICALIZE:
105  R = ScalarizeVecRes_UnaryOp(N);
106  break;
107 
108  case ISD::ADD:
109  case ISD::AND:
110  case ISD::FADD:
111  case ISD::FCOPYSIGN:
112  case ISD::FDIV:
113  case ISD::FMUL:
114  case ISD::FMINNUM:
115  case ISD::FMAXNUM:
116  case ISD::FMINNAN:
117  case ISD::FMAXNAN:
118  case ISD::SMIN:
119  case ISD::SMAX:
120  case ISD::UMIN:
121  case ISD::UMAX:
122 
123  case ISD::FPOW:
124  case ISD::FREM:
125  case ISD::FSUB:
126  case ISD::MUL:
127  case ISD::OR:
128  case ISD::SDIV:
129  case ISD::SREM:
130  case ISD::SUB:
131  case ISD::UDIV:
132  case ISD::UREM:
133  case ISD::XOR:
134  case ISD::SHL:
135  case ISD::SRA:
136  case ISD::SRL:
137  R = ScalarizeVecRes_BinOp(N);
138  break;
139  case ISD::FMA:
140  R = ScalarizeVecRes_TernaryOp(N);
141  break;
142  }
143 
144  // If R is null, the sub-method took care of registering the result.
145  if (R.getNode())
146  SetScalarizedVector(SDValue(N, ResNo), R);
147 }
148 
149 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
150  SDValue LHS = GetScalarizedVector(N->getOperand(0));
151  SDValue RHS = GetScalarizedVector(N->getOperand(1));
152  return DAG.getNode(N->getOpcode(), SDLoc(N),
153  LHS.getValueType(), LHS, RHS, N->getFlags());
154 }
155 
156 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
157  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
158  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
159  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
160  return DAG.getNode(N->getOpcode(), SDLoc(N),
161  Op0.getValueType(), Op0, Op1, Op2);
162 }
163 
164 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
165  unsigned ResNo) {
166  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
167  return GetScalarizedVector(Op);
168 }
169 
170 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
171  SDValue Op = N->getOperand(0);
172  if (Op.getValueType().isVector()
173  && Op.getValueType().getVectorNumElements() == 1
174  && !isSimpleLegalType(Op.getValueType()))
175  Op = GetScalarizedVector(Op);
176  EVT NewVT = N->getValueType(0).getVectorElementType();
177  return DAG.getNode(ISD::BITCAST, SDLoc(N),
178  NewVT, Op);
179 }
180 
181 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
182  EVT EltVT = N->getValueType(0).getVectorElementType();
183  SDValue InOp = N->getOperand(0);
184  // The BUILD_VECTOR operands may be of wider element types and
185  // we may need to truncate them back to the requested return type.
186  if (EltVT.isInteger())
187  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
188  return InOp;
189 }
190 
191 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
192  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
194  N->getOperand(0), N->getOperand(1));
195 }
196 
197 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
198  EVT NewVT = N->getValueType(0).getVectorElementType();
199  SDValue Op = GetScalarizedVector(N->getOperand(0));
200  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
201  NewVT, Op, N->getOperand(1));
202 }
203 
204 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
205  SDValue Op = GetScalarizedVector(N->getOperand(0));
206  return DAG.getNode(ISD::FPOWI, SDLoc(N),
207  Op.getValueType(), Op, N->getOperand(1));
208 }
209 
210 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
211  // The value to insert may have a wider type than the vector element type,
212  // so be sure to truncate it to the element type if necessary.
213  SDValue Op = N->getOperand(1);
214  EVT EltVT = N->getValueType(0).getVectorElementType();
215  if (Op.getValueType() != EltVT)
216  // FIXME: Can this happen for floating point types?
217  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
218  return Op;
219 }
220 
221 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
222  assert(N->isUnindexed() && "Indexed vector load?");
223 
224  SDValue Result = DAG.getLoad(
227  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
230  N->getAAInfo());
231 
232  // Legalize the chain result - switch anything that used the old chain to
233  // use the new one.
234  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
235  return Result;
236 }
237 
238 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
239  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
240  EVT DestVT = N->getValueType(0).getVectorElementType();
241  SDValue Op = N->getOperand(0);
242  EVT OpVT = Op.getValueType();
243  SDLoc DL(N);
244  // The result needs scalarizing, but it's not a given that the source does.
245  // This is a workaround for targets where it's impossible to scalarize the
246  // result of a conversion, because the source type is legal.
247  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
248  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
249  // legal and was not scalarized.
250  // See the similar logic in ScalarizeVecRes_SETCC
251  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
252  Op = GetScalarizedVector(Op);
253  } else {
254  EVT VT = OpVT.getVectorElementType();
255  Op = DAG.getNode(
256  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
257  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
258  }
259  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
260 }
261 
262 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
263  EVT EltVT = N->getValueType(0).getVectorElementType();
264  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
265  SDValue LHS = GetScalarizedVector(N->getOperand(0));
266  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
267  LHS, DAG.getValueType(ExtVT));
268 }
269 
270 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
271  SDLoc DL(N);
272  SDValue Op = N->getOperand(0);
273 
274  EVT OpVT = Op.getValueType();
275  EVT OpEltVT = OpVT.getVectorElementType();
276  EVT EltVT = N->getValueType(0).getVectorElementType();
277 
278  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
279  Op = GetScalarizedVector(Op);
280  } else {
281  Op = DAG.getNode(
282  ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
283  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
284  }
285 
286  switch (N->getOpcode()) {
288  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
290  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
292  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
293  }
294 
295  llvm_unreachable("Illegal extend_vector_inreg opcode");
296 }
297 
298 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
299  // If the operand is wider than the vector element type then it is implicitly
300  // truncated. Make that explicit here.
301  EVT EltVT = N->getValueType(0).getVectorElementType();
302  SDValue InOp = N->getOperand(0);
303  if (InOp.getValueType() != EltVT)
304  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
305  return InOp;
306 }
307 
308 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
309  SDValue Cond = N->getOperand(0);
310  EVT OpVT = Cond.getValueType();
311  SDLoc DL(N);
312  // The vselect result and true/value operands needs scalarizing, but it's
313  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
314  // See the similar logic in ScalarizeVecRes_SETCC
315  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
316  Cond = GetScalarizedVector(Cond);
317  } else {
318  EVT VT = OpVT.getVectorElementType();
319  Cond = DAG.getNode(
320  ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
321  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
322  }
323 
324  SDValue LHS = GetScalarizedVector(N->getOperand(1));
325  TargetLowering::BooleanContent ScalarBool =
326  TLI.getBooleanContents(false, false);
327  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
328 
329  // If integer and float booleans have different contents then we can't
330  // reliably optimize in all cases. There is a full explanation for this in
331  // DAGCombiner::visitSELECT() where the same issue affects folding
332  // (select C, 0, 1) to (xor C, 1).
333  if (TLI.getBooleanContents(false, false) !=
334  TLI.getBooleanContents(false, true)) {
335  // At least try the common case where the boolean is generated by a
336  // comparison.
337  if (Cond->getOpcode() == ISD::SETCC) {
338  EVT OpVT = Cond->getOperand(0).getValueType();
339  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
340  VecBool = TLI.getBooleanContents(OpVT);
341  } else
343  }
344 
345  EVT CondVT = Cond.getValueType();
346  if (ScalarBool != VecBool) {
347  switch (ScalarBool) {
349  break;
353  // Vector read from all ones, scalar expects a single 1 so mask.
354  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
355  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
356  break;
360  // Vector reads from a one, scalar from all ones so sign extend.
361  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
362  Cond, DAG.getValueType(MVT::i1));
363  break;
364  }
365  }
366 
367  // Truncate the condition if needed
368  auto BoolVT = getSetCCResultType(CondVT);
369  if (BoolVT.bitsLT(CondVT))
370  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
371 
372  return DAG.getSelect(SDLoc(N),
373  LHS.getValueType(), Cond, LHS,
374  GetScalarizedVector(N->getOperand(2)));
375 }
376 
377 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
378  SDValue LHS = GetScalarizedVector(N->getOperand(1));
379  return DAG.getSelect(SDLoc(N),
380  LHS.getValueType(), N->getOperand(0), LHS,
381  GetScalarizedVector(N->getOperand(2)));
382 }
383 
384 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
385  SDValue LHS = GetScalarizedVector(N->getOperand(2));
386  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
387  N->getOperand(0), N->getOperand(1),
388  LHS, GetScalarizedVector(N->getOperand(3)),
389  N->getOperand(4));
390 }
391 
392 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
393  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
394 }
395 
396 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
397  // Figure out if the scalar is the LHS or RHS and return it.
398  SDValue Arg = N->getOperand(2).getOperand(0);
399  if (Arg.isUndef())
400  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
401  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
402  return GetScalarizedVector(N->getOperand(Op));
403 }
404 
405 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
406  assert(N->getValueType(0).isVector() &&
407  N->getOperand(0).getValueType().isVector() &&
408  "Operand types must be vectors");
409  SDValue LHS = N->getOperand(0);
410  SDValue RHS = N->getOperand(1);
411  EVT OpVT = LHS.getValueType();
412  EVT NVT = N->getValueType(0).getVectorElementType();
413  SDLoc DL(N);
414 
415  // The result needs scalarizing, but it's not a given that the source does.
416  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
417  LHS = GetScalarizedVector(LHS);
418  RHS = GetScalarizedVector(RHS);
419  } else {
420  EVT VT = OpVT.getVectorElementType();
421  LHS = DAG.getNode(
422  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
423  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
424  RHS = DAG.getNode(
425  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
426  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
427  }
428 
429  // Turn it into a scalar SETCC.
430  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
431  N->getOperand(2));
432  // Vectors may have a different boolean contents to scalars. Promote the
433  // value appropriately.
434  ISD::NodeType ExtendCode =
436  return DAG.getNode(ExtendCode, DL, NVT, Res);
437 }
438 
439 
440 //===----------------------------------------------------------------------===//
441 // Operand Vector Scalarization <1 x ty> -> ty.
442 //===----------------------------------------------------------------------===//
443 
444 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
445  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
446  dbgs() << "\n");
447  SDValue Res = SDValue();
448 
449  if (!Res.getNode()) {
450  switch (N->getOpcode()) {
451  default:
452 #ifndef NDEBUG
453  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
454  N->dump(&DAG);
455  dbgs() << "\n";
456 #endif
457  report_fatal_error("Do not know how to scalarize this operator's "
458  "operand!\n");
459  case ISD::BITCAST:
460  Res = ScalarizeVecOp_BITCAST(N);
461  break;
462  case ISD::ANY_EXTEND:
463  case ISD::ZERO_EXTEND:
464  case ISD::SIGN_EXTEND:
465  case ISD::TRUNCATE:
466  case ISD::FP_TO_SINT:
467  case ISD::FP_TO_UINT:
468  case ISD::SINT_TO_FP:
469  case ISD::UINT_TO_FP:
470  Res = ScalarizeVecOp_UnaryOp(N);
471  break;
472  case ISD::CONCAT_VECTORS:
473  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
474  break;
476  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
477  break;
478  case ISD::VSELECT:
479  Res = ScalarizeVecOp_VSELECT(N);
480  break;
481  case ISD::SETCC:
482  Res = ScalarizeVecOp_VSETCC(N);
483  break;
484  case ISD::STORE:
485  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
486  break;
487  case ISD::FP_ROUND:
488  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
489  break;
490  }
491  }
492 
493  // If the result is null, the sub-method took care of registering results etc.
494  if (!Res.getNode()) return false;
495 
496  // If the result is N, the sub-method updated N in place. Tell the legalizer
497  // core about this.
498  if (Res.getNode() == N)
499  return true;
500 
501  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
502  "Invalid operand expansion");
503 
504  ReplaceValueWith(SDValue(N, 0), Res);
505  return false;
506 }
507 
508 /// If the value to convert is a vector that needs to be scalarized, it must be
509 /// <1 x ty>. Convert the element instead.
510 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
511  SDValue Elt = GetScalarizedVector(N->getOperand(0));
512  return DAG.getNode(ISD::BITCAST, SDLoc(N),
513  N->getValueType(0), Elt);
514 }
515 
516 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
517 /// Do the operation on the element instead.
518 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
520  "Unexpected vector type!");
521  SDValue Elt = GetScalarizedVector(N->getOperand(0));
522  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
523  N->getValueType(0).getScalarType(), Elt);
524  // Revectorize the result so the types line up with what the uses of this
525  // expression expect.
526  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
527 }
528 
529 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
530 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
532  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
533  Ops[i] = GetScalarizedVector(N->getOperand(i));
534  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
535 }
536 
537 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
538 /// so just return the element, ignoring the index.
539 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
540  EVT VT = N->getValueType(0);
541  SDValue Res = GetScalarizedVector(N->getOperand(0));
542  if (Res.getValueType() != VT)
543  Res = VT.isFloatingPoint()
544  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
545  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
546  return Res;
547 }
548 
549 /// If the input condition is a vector that needs to be scalarized, it must be
550 /// <1 x i1>, so just convert to a normal ISD::SELECT
551 /// (still with vector output type since that was acceptable if we got here).
552 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
553  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
554  EVT VT = N->getValueType(0);
555 
556  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
557  N->getOperand(2));
558 }
559 
560 /// If the operand is a vector that needs to be scalarized then the
561 /// result must be v1i1, so just convert to a scalar SETCC and wrap
562 /// with a scalar_to_vector since the res type is legal if we got here
563 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
564  assert(N->getValueType(0).isVector() &&
565  N->getOperand(0).getValueType().isVector() &&
566  "Operand types must be vectors");
567  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
568 
569  EVT VT = N->getValueType(0);
570  SDValue LHS = GetScalarizedVector(N->getOperand(0));
571  SDValue RHS = GetScalarizedVector(N->getOperand(1));
572 
573  EVT OpVT = N->getOperand(0).getValueType();
574  EVT NVT = VT.getVectorElementType();
575  SDLoc DL(N);
576  // Turn it into a scalar SETCC.
577  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
578  N->getOperand(2));
579 
580  // Vectors may have a different boolean contents to scalars. Promote the
581  // value appropriately.
582  ISD::NodeType ExtendCode =
584 
585  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
586 
587  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
588 }
589 
590 /// If the value to store is a vector that needs to be scalarized, it must be
591 /// <1 x ty>. Just store the element.
592 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
593  assert(N->isUnindexed() && "Indexed store of one-element vector?");
594  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
595  SDLoc dl(N);
596 
597  if (N->isTruncatingStore())
598  return DAG.getTruncStore(
599  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
600  N->getBasePtr(), N->getPointerInfo(),
602  N->getMemOperand()->getFlags(), N->getAAInfo());
603 
604  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
605  N->getBasePtr(), N->getPointerInfo(),
607  N->getAAInfo());
608 }
609 
610 /// If the value to round is a vector that needs to be scalarized, it must be
611 /// <1 x ty>. Convert the element instead.
612 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
613  SDValue Elt = GetScalarizedVector(N->getOperand(0));
614  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
615  N->getValueType(0).getVectorElementType(), Elt,
616  N->getOperand(1));
617  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
618 }
619 
620 //===----------------------------------------------------------------------===//
621 // Result Vector Splitting
622 //===----------------------------------------------------------------------===//
623 
624 /// This method is called when the specified result of the specified node is
625 /// found to need vector splitting. At this point, the node may also have
626 /// invalid operands or may have other results that need legalization, we just
627 /// know that (at least) one result needs vector splitting.
628 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
629  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
630  SDValue Lo, Hi;
631 
632  // See if the target wants to custom expand this node.
633  if (CustomLowerNode(N, N->getValueType(ResNo), true))
634  return;
635 
636  switch (N->getOpcode()) {
637  default:
638 #ifndef NDEBUG
639  dbgs() << "SplitVectorResult #" << ResNo << ": ";
640  N->dump(&DAG);
641  dbgs() << "\n";
642 #endif
643  report_fatal_error("Do not know how to split the result of this "
644  "operator!\n");
645 
646  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
647  case ISD::VSELECT:
648  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
649  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
650  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
651  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
652  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
653  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
654  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
655  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
656  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
657  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
658  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
659  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
660  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
661  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
662  case ISD::LOAD:
663  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
664  break;
665  case ISD::MLOAD:
666  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
667  break;
668  case ISD::MGATHER:
669  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
670  break;
671  case ISD::SETCC:
672  SplitVecRes_SETCC(N, Lo, Hi);
673  break;
674  case ISD::VECTOR_SHUFFLE:
675  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
676  break;
677 
681  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
682  break;
683 
684  case ISD::BITREVERSE:
685  case ISD::BSWAP:
686  case ISD::CTLZ:
687  case ISD::CTTZ:
690  case ISD::CTPOP:
691  case ISD::FABS:
692  case ISD::FCEIL:
693  case ISD::FCOS:
694  case ISD::FEXP:
695  case ISD::FEXP2:
696  case ISD::FFLOOR:
697  case ISD::FLOG:
698  case ISD::FLOG10:
699  case ISD::FLOG2:
700  case ISD::FNEARBYINT:
701  case ISD::FNEG:
702  case ISD::FP_EXTEND:
703  case ISD::FP_ROUND:
704  case ISD::FP_TO_SINT:
705  case ISD::FP_TO_UINT:
706  case ISD::FRINT:
707  case ISD::FROUND:
708  case ISD::FSIN:
709  case ISD::FSQRT:
710  case ISD::FTRUNC:
711  case ISD::SINT_TO_FP:
712  case ISD::TRUNCATE:
713  case ISD::UINT_TO_FP:
714  case ISD::FCANONICALIZE:
715  SplitVecRes_UnaryOp(N, Lo, Hi);
716  break;
717 
718  case ISD::ANY_EXTEND:
719  case ISD::SIGN_EXTEND:
720  case ISD::ZERO_EXTEND:
721  SplitVecRes_ExtendOp(N, Lo, Hi);
722  break;
723 
724  case ISD::ADD:
725  case ISD::SUB:
726  case ISD::MUL:
727  case ISD::MULHS:
728  case ISD::MULHU:
729  case ISD::FADD:
730  case ISD::FSUB:
731  case ISD::FMUL:
732  case ISD::FMINNUM:
733  case ISD::FMAXNUM:
734  case ISD::FMINNAN:
735  case ISD::FMAXNAN:
736  case ISD::SDIV:
737  case ISD::UDIV:
738  case ISD::FDIV:
739  case ISD::FPOW:
740  case ISD::AND:
741  case ISD::OR:
742  case ISD::XOR:
743  case ISD::SHL:
744  case ISD::SRA:
745  case ISD::SRL:
746  case ISD::UREM:
747  case ISD::SREM:
748  case ISD::FREM:
749  case ISD::SMIN:
750  case ISD::SMAX:
751  case ISD::UMIN:
752  case ISD::UMAX:
753  SplitVecRes_BinOp(N, Lo, Hi);
754  break;
755  case ISD::FMA:
756  SplitVecRes_TernaryOp(N, Lo, Hi);
757  break;
758  }
759 
760  // If Lo/Hi is null, the sub-method took care of registering results etc.
761  if (Lo.getNode())
762  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
763 }
764 
765 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
766  SDValue &Hi) {
767  SDValue LHSLo, LHSHi;
768  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
769  SDValue RHSLo, RHSHi;
770  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
771  SDLoc dl(N);
772 
773  const SDNodeFlags Flags = N->getFlags();
774  unsigned Opcode = N->getOpcode();
775  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
776  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
777 }
778 
779 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
780  SDValue &Hi) {
781  SDValue Op0Lo, Op0Hi;
782  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
783  SDValue Op1Lo, Op1Hi;
784  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
785  SDValue Op2Lo, Op2Hi;
786  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
787  SDLoc dl(N);
788 
789  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
790  Op0Lo, Op1Lo, Op2Lo);
791  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
792  Op0Hi, Op1Hi, Op2Hi);
793 }
794 
795 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
796  SDValue &Hi) {
797  // We know the result is a vector. The input may be either a vector or a
798  // scalar value.
799  EVT LoVT, HiVT;
800  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
801  SDLoc dl(N);
802 
803  SDValue InOp = N->getOperand(0);
804  EVT InVT = InOp.getValueType();
805 
806  // Handle some special cases efficiently.
807  switch (getTypeAction(InVT)) {
814  break;
817  // A scalar to vector conversion, where the scalar needs expansion.
818  // If the vector is being split in two then we can just convert the
819  // expanded pieces.
820  if (LoVT == HiVT) {
821  GetExpandedOp(InOp, Lo, Hi);
822  if (DAG.getDataLayout().isBigEndian())
823  std::swap(Lo, Hi);
824  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
825  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
826  return;
827  }
828  break;
830  // If the input is a vector that needs to be split, convert each split
831  // piece of the input now.
832  GetSplitVector(InOp, Lo, Hi);
833  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
834  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
835  return;
836  }
837 
838  // In the general case, convert the input to an integer and split it by hand.
839  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
840  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
841  if (DAG.getDataLayout().isBigEndian())
842  std::swap(LoIntVT, HiIntVT);
843 
844  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
845 
846  if (DAG.getDataLayout().isBigEndian())
847  std::swap(Lo, Hi);
848  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
849  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
850 }
851 
852 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
853  SDValue &Hi) {
854  EVT LoVT, HiVT;
855  SDLoc dl(N);
856  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
857  unsigned LoNumElts = LoVT.getVectorNumElements();
858  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
859  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
860 
861  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
862  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
863 }
864 
865 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
866  SDValue &Hi) {
867  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
868  SDLoc dl(N);
869  unsigned NumSubvectors = N->getNumOperands() / 2;
870  if (NumSubvectors == 1) {
871  Lo = N->getOperand(0);
872  Hi = N->getOperand(1);
873  return;
874  }
875 
876  EVT LoVT, HiVT;
877  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
878 
879  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
880  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
881 
882  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
883  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
884 }
885 
886 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
887  SDValue &Hi) {
888  SDValue Vec = N->getOperand(0);
889  SDValue Idx = N->getOperand(1);
890  SDLoc dl(N);
891 
892  EVT LoVT, HiVT;
893  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
894 
895  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
896  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
897  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
898  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
899  TLI.getVectorIdxTy(DAG.getDataLayout())));
900 }
901 
902 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
903  SDValue &Hi) {
904  SDValue Vec = N->getOperand(0);
905  SDValue SubVec = N->getOperand(1);
906  SDValue Idx = N->getOperand(2);
907  SDLoc dl(N);
908  GetSplitVector(Vec, Lo, Hi);
909 
910  EVT VecVT = Vec.getValueType();
911  unsigned VecElems = VecVT.getVectorNumElements();
912  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
913 
914  // If we know the index is 0, and we know the subvector doesn't cross the
915  // boundary between the halves, we can avoid spilling the vector, and insert
916  // into the lower half of the split vector directly.
917  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
918  // the index is constant and there is no boundary crossing. But those cases
919  // don't seem to get hit in practice.
920  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
921  unsigned IdxVal = ConstIdx->getZExtValue();
922  if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
923  EVT LoVT, HiVT;
924  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
925  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
926  return;
927  }
928  }
929 
930  // Spill the vector to the stack.
931  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
932  SDValue Store =
933  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
934 
935  // Store the new subvector into the specified index.
936  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
937  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
938  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
939  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
940 
941  // Load the Lo part from the stack slot.
942  Lo =
943  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
944 
945  // Increment the pointer to the other part.
946  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
947  StackPtr =
948  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
949  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
950 
951  // Load the Hi part from the stack slot.
952  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
953  MinAlign(Alignment, IncrementSize));
954 }
955 
956 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
957  SDValue &Hi) {
958  SDLoc dl(N);
959  GetSplitVector(N->getOperand(0), Lo, Hi);
960  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
961  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
962 }
963 
964 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
965  SDValue &Hi) {
966  SDValue LHSLo, LHSHi;
967  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
968  SDLoc DL(N);
969 
970  SDValue RHSLo, RHSHi;
971  SDValue RHS = N->getOperand(1);
972  EVT RHSVT = RHS.getValueType();
973  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
974  GetSplitVector(RHS, RHSLo, RHSHi);
975  else
976  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
977 
978 
979  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
980  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
981 }
982 
983 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
984  SDValue &Hi) {
985  SDValue LHSLo, LHSHi;
986  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
987  SDLoc dl(N);
988 
989  EVT LoVT, HiVT;
990  std::tie(LoVT, HiVT) =
991  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
992 
993  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
994  DAG.getValueType(LoVT));
995  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
996  DAG.getValueType(HiVT));
997 }
998 
999 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1000  SDValue &Hi) {
1001  unsigned Opcode = N->getOpcode();
1002  SDValue N0 = N->getOperand(0);
1003 
1004  SDLoc dl(N);
1005  SDValue InLo, InHi;
1006 
1007  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1008  GetSplitVector(N0, InLo, InHi);
1009  else
1010  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1011 
1012  EVT InLoVT = InLo.getValueType();
1013  unsigned InNumElements = InLoVT.getVectorNumElements();
1014 
1015  EVT OutLoVT, OutHiVT;
1016  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1017  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1018  assert((2 * OutNumElements) <= InNumElements &&
1019  "Illegal extend vector in reg split");
1020 
1021  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1022  // input vector (i.e. we only use InLo):
1023  // OutLo will extend the first OutNumElements from InLo.
1024  // OutHi will extend the next OutNumElements from InLo.
1025 
1026  // Shuffle the elements from InLo for OutHi into the bottom elements to
1027  // create a 'fake' InHi.
1028  SmallVector<int, 8> SplitHi(InNumElements, -1);
1029  for (unsigned i = 0; i != OutNumElements; ++i)
1030  SplitHi[i] = i + OutNumElements;
1031  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1032 
1033  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1034  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1035 }
1036 
1037 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1038  SDValue &Hi) {
1039  SDValue Vec = N->getOperand(0);
1040  SDValue Elt = N->getOperand(1);
1041  SDValue Idx = N->getOperand(2);
1042  SDLoc dl(N);
1043  GetSplitVector(Vec, Lo, Hi);
1044 
1045  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1046  unsigned IdxVal = CIdx->getZExtValue();
1047  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1048  if (IdxVal < LoNumElts)
1049  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1050  Lo.getValueType(), Lo, Elt, Idx);
1051  else
1052  Hi =
1053  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1054  DAG.getConstant(IdxVal - LoNumElts, dl,
1055  TLI.getVectorIdxTy(DAG.getDataLayout())));
1056  return;
1057  }
1058 
1059  // See if the target wants to custom expand this node.
1060  if (CustomLowerNode(N, N->getValueType(0), true))
1061  return;
1062 
1063  // Make the vector elements byte-addressable if they aren't already.
1064  EVT VecVT = Vec.getValueType();
1065  EVT EltVT = VecVT.getVectorElementType();
1066  if (VecVT.getScalarSizeInBits() < 8) {
1067  EltVT = MVT::i8;
1068  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1069  VecVT.getVectorNumElements());
1070  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1071  // Extend the element type to match if needed.
1072  if (EltVT.bitsGT(Elt.getValueType()))
1073  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1074  }
1075 
1076  // Spill the vector to the stack.
1077  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1078  auto &MF = DAG.getMachineFunction();
1079  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1080  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1081  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1082 
1083  // Store the new element. This may be larger than the vector element type,
1084  // so use a truncating store.
1085  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1086  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1087  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1088  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1090 
1091  EVT LoVT, HiVT;
1092  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1093 
1094  // Load the Lo part from the stack slot.
1095  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1096 
1097  // Increment the pointer to the other part.
1098  unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1099  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1100  DAG.getConstant(IncrementSize, dl,
1101  StackPtr.getValueType()));
1102 
1103  // Load the Hi part from the stack slot.
1104  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1105  PtrInfo.getWithOffset(IncrementSize),
1106  MinAlign(Alignment, IncrementSize));
1107 
1108  // If we adjusted the original type, we need to truncate the results.
1109  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1110  if (LoVT != Lo.getValueType())
1111  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1112  if (HiVT != Hi.getValueType())
1113  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1114 }
1115 
1116 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1117  SDValue &Hi) {
1118  EVT LoVT, HiVT;
1119  SDLoc dl(N);
1120  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1121  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1122  Hi = DAG.getUNDEF(HiVT);
1123 }
1124 
1125 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1126  SDValue &Hi) {
1127  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1128  EVT LoVT, HiVT;
1129  SDLoc dl(LD);
1130  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1131 
1132  ISD::LoadExtType ExtType = LD->getExtensionType();
1133  SDValue Ch = LD->getChain();
1134  SDValue Ptr = LD->getBasePtr();
1135  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1136  EVT MemoryVT = LD->getMemoryVT();
1137  unsigned Alignment = LD->getOriginalAlignment();
1138  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1139  AAMDNodes AAInfo = LD->getAAInfo();
1140 
1141  EVT LoMemVT, HiMemVT;
1142  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1143 
1144  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1145  LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1146 
1147  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1148  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1149  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1150  LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1151  Alignment, MMOFlags, AAInfo);
1152 
1153  // Build a factor node to remember that this load is independent of the
1154  // other one.
1155  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1156  Hi.getValue(1));
1157 
1158  // Legalize the chain result - switch anything that used the old chain to
1159  // use the new one.
1160  ReplaceValueWith(SDValue(LD, 1), Ch);
1161 }
1162 
1163 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1164  SDValue &Lo, SDValue &Hi) {
1165  EVT LoVT, HiVT;
1166  SDLoc dl(MLD);
1167  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1168 
1169  SDValue Ch = MLD->getChain();
1170  SDValue Ptr = MLD->getBasePtr();
1171  SDValue Mask = MLD->getMask();
1172  SDValue Src0 = MLD->getSrc0();
1173  unsigned Alignment = MLD->getOriginalAlignment();
1174  ISD::LoadExtType ExtType = MLD->getExtensionType();
1175 
1176  // if Alignment is equal to the vector size,
1177  // take the half of it for the second part
1178  unsigned SecondHalfAlignment =
1179  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1180  Alignment/2 : Alignment;
1181 
1182  // Split Mask operand
1183  SDValue MaskLo, MaskHi;
1184  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1185  GetSplitVector(Mask, MaskLo, MaskHi);
1186  else
1187  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1188 
1189  EVT MemoryVT = MLD->getMemoryVT();
1190  EVT LoMemVT, HiMemVT;
1191  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1192 
1193  SDValue Src0Lo, Src0Hi;
1194  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1195  GetSplitVector(Src0, Src0Lo, Src0Hi);
1196  else
1197  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1198 
1200  getMachineMemOperand(MLD->getPointerInfo(),
1202  Alignment, MLD->getAAInfo(), MLD->getRanges());
1203 
1204  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
1205  ExtType, MLD->isExpandingLoad());
1206 
1207  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1208  MLD->isExpandingLoad());
1209  unsigned HiOffset = LoMemVT.getStoreSize();
1210 
1213  HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
1214  MLD->getRanges());
1215 
1216  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
1217  ExtType, MLD->isExpandingLoad());
1218 
1219  // Build a factor node to remember that this load is independent of the
1220  // other one.
1221  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1222  Hi.getValue(1));
1223 
1224  // Legalize the chain result - switch anything that used the old chain to
1225  // use the new one.
1226  ReplaceValueWith(SDValue(MLD, 1), Ch);
1227 
1228 }
1229 
1230 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1231  SDValue &Lo, SDValue &Hi) {
1232  EVT LoVT, HiVT;
1233  SDLoc dl(MGT);
1234  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1235 
1236  SDValue Ch = MGT->getChain();
1237  SDValue Ptr = MGT->getBasePtr();
1238  SDValue Mask = MGT->getMask();
1239  SDValue Src0 = MGT->getValue();
1240  SDValue Index = MGT->getIndex();
1241  SDValue Scale = MGT->getScale();
1242  unsigned Alignment = MGT->getOriginalAlignment();
1243 
1244  // Split Mask operand
1245  SDValue MaskLo, MaskHi;
1246  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1247  GetSplitVector(Mask, MaskLo, MaskHi);
1248  else
1249  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1250 
1251  EVT MemoryVT = MGT->getMemoryVT();
1252  EVT LoMemVT, HiMemVT;
1253  // Split MemoryVT
1254  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1255 
1256  SDValue Src0Lo, Src0Hi;
1257  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1258  GetSplitVector(Src0, Src0Lo, Src0Hi);
1259  else
1260  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1261 
1262  SDValue IndexHi, IndexLo;
1263  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1264  GetSplitVector(Index, IndexLo, IndexHi);
1265  else
1266  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1267 
1269  getMachineMemOperand(MGT->getPointerInfo(),
1271  Alignment, MGT->getAAInfo(), MGT->getRanges());
1272 
1273  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
1274  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1275  MMO);
1276 
1277  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
1278  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1279  MMO);
1280 
1281  // Build a factor node to remember that this load is independent of the
1282  // other one.
1283  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1284  Hi.getValue(1));
1285 
1286  // Legalize the chain result - switch anything that used the old chain to
1287  // use the new one.
1288  ReplaceValueWith(SDValue(MGT, 1), Ch);
1289 }
1290 
1291 
1292 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1293  assert(N->getValueType(0).isVector() &&
1294  N->getOperand(0).getValueType().isVector() &&
1295  "Operand types must be vectors");
1296 
1297  EVT LoVT, HiVT;
1298  SDLoc DL(N);
1299  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1300 
1301  // If the input also splits, handle it directly. Otherwise split it by hand.
1302  SDValue LL, LH, RL, RH;
1303  if (getTypeAction(N->getOperand(0).getValueType()) ==
1305  GetSplitVector(N->getOperand(0), LL, LH);
1306  else
1307  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1308 
1309  if (getTypeAction(N->getOperand(1).getValueType()) ==
1311  GetSplitVector(N->getOperand(1), RL, RH);
1312  else
1313  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1314 
1315  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1316  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1317 }
1318 
1319 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1320  SDValue &Hi) {
1321  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1322  EVT LoVT, HiVT;
1323  SDLoc dl(N);
1324  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1325 
1326  // If the input also splits, handle it directly for a compile time speedup.
1327  // Otherwise split it by hand.
1328  EVT InVT = N->getOperand(0).getValueType();
1329  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1330  GetSplitVector(N->getOperand(0), Lo, Hi);
1331  else
1332  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1333 
1334  if (N->getOpcode() == ISD::FP_ROUND) {
1335  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1336  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1337  } else {
1338  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1339  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1340  }
1341 }
1342 
1343 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1344  SDValue &Hi) {
1345  SDLoc dl(N);
1346  EVT SrcVT = N->getOperand(0).getValueType();
1347  EVT DestVT = N->getValueType(0);
1348  EVT LoVT, HiVT;
1349  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1350 
1351  // We can do better than a generic split operation if the extend is doing
1352  // more than just doubling the width of the elements and the following are
1353  // true:
1354  // - The number of vector elements is even,
1355  // - the source type is legal,
1356  // - the type of a split source is illegal,
1357  // - the type of an extended (by doubling element size) source is legal, and
1358  // - the type of that extended source when split is legal.
1359  //
1360  // This won't necessarily completely legalize the operation, but it will
1361  // more effectively move in the right direction and prevent falling down
1362  // to scalarization in many cases due to the input vector being split too
1363  // far.
1364  unsigned NumElements = SrcVT.getVectorNumElements();
1365  if ((NumElements & 1) == 0 &&
1366  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1367  LLVMContext &Ctx = *DAG.getContext();
1368  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1369  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1370 
1371  EVT SplitLoVT, SplitHiVT;
1372  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1373  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1374  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1375  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1376  N->dump(&DAG); dbgs() << "\n");
1377  // Extend the source vector by one step.
1378  SDValue NewSrc =
1379  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1380  // Get the low and high halves of the new, extended one step, vector.
1381  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1382  // Extend those vector halves the rest of the way.
1383  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1384  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1385  return;
1386  }
1387  }
1388  // Fall back to the generic unary operator splitting otherwise.
1389  SplitVecRes_UnaryOp(N, Lo, Hi);
1390 }
1391 
1392 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1393  SDValue &Lo, SDValue &Hi) {
1394  // The low and high parts of the original input give four input vectors.
1395  SDValue Inputs[4];
1396  SDLoc dl(N);
1397  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1398  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1399  EVT NewVT = Inputs[0].getValueType();
1400  unsigned NewElts = NewVT.getVectorNumElements();
1401 
1402  // If Lo or Hi uses elements from at most two of the four input vectors, then
1403  // express it as a vector shuffle of those two inputs. Otherwise extract the
1404  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1406  for (unsigned High = 0; High < 2; ++High) {
1407  SDValue &Output = High ? Hi : Lo;
1408 
1409  // Build a shuffle mask for the output, discovering on the fly which
1410  // input vectors to use as shuffle operands (recorded in InputUsed).
1411  // If building a suitable shuffle vector proves too hard, then bail
1412  // out with useBuildVector set.
1413  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1414  unsigned FirstMaskIdx = High * NewElts;
1415  bool useBuildVector = false;
1416  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1417  // The mask element. This indexes into the input.
1418  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1419 
1420  // The input vector this mask element indexes into.
1421  unsigned Input = (unsigned)Idx / NewElts;
1422 
1423  if (Input >= array_lengthof(Inputs)) {
1424  // The mask element does not index into any input vector.
1425  Ops.push_back(-1);
1426  continue;
1427  }
1428 
1429  // Turn the index into an offset from the start of the input vector.
1430  Idx -= Input * NewElts;
1431 
1432  // Find or create a shuffle vector operand to hold this input.
1433  unsigned OpNo;
1434  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1435  if (InputUsed[OpNo] == Input) {
1436  // This input vector is already an operand.
1437  break;
1438  } else if (InputUsed[OpNo] == -1U) {
1439  // Create a new operand for this input vector.
1440  InputUsed[OpNo] = Input;
1441  break;
1442  }
1443  }
1444 
1445  if (OpNo >= array_lengthof(InputUsed)) {
1446  // More than two input vectors used! Give up on trying to create a
1447  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1448  useBuildVector = true;
1449  break;
1450  }
1451 
1452  // Add the mask index for the new shuffle vector.
1453  Ops.push_back(Idx + OpNo * NewElts);
1454  }
1455 
1456  if (useBuildVector) {
1457  EVT EltVT = NewVT.getVectorElementType();
1459 
1460  // Extract the input elements by hand.
1461  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1462  // The mask element. This indexes into the input.
1463  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1464 
1465  // The input vector this mask element indexes into.
1466  unsigned Input = (unsigned)Idx / NewElts;
1467 
1468  if (Input >= array_lengthof(Inputs)) {
1469  // The mask element is "undef" or indexes off the end of the input.
1470  SVOps.push_back(DAG.getUNDEF(EltVT));
1471  continue;
1472  }
1473 
1474  // Turn the index into an offset from the start of the input vector.
1475  Idx -= Input * NewElts;
1476 
1477  // Extract the vector element by hand.
1478  SVOps.push_back(DAG.getNode(
1479  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1480  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1481  }
1482 
1483  // Construct the Lo/Hi output using a BUILD_VECTOR.
1484  Output = DAG.getBuildVector(NewVT, dl, SVOps);
1485  } else if (InputUsed[0] == -1U) {
1486  // No input vectors were used! The result is undefined.
1487  Output = DAG.getUNDEF(NewVT);
1488  } else {
1489  SDValue Op0 = Inputs[InputUsed[0]];
1490  // If only one input was used, use an undefined vector for the other.
1491  SDValue Op1 = InputUsed[1] == -1U ?
1492  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1493  // At least one input vector was used. Create a new shuffle vector.
1494  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1495  }
1496 
1497  Ops.clear();
1498  }
1499 }
1500 
1501 
1502 //===----------------------------------------------------------------------===//
1503 // Operand Vector Splitting
1504 //===----------------------------------------------------------------------===//
1505 
1506 /// This method is called when the specified operand of the specified node is
1507 /// found to need vector splitting. At this point, all of the result types of
1508 /// the node are known to be legal, but other operands of the node may need
1509 /// legalization as well as the specified one.
1510 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1511  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1512  SDValue Res = SDValue();
1513 
1514  // See if the target wants to custom split this node.
1515  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1516  return false;
1517 
1518  if (!Res.getNode()) {
1519  switch (N->getOpcode()) {
1520  default:
1521 #ifndef NDEBUG
1522  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1523  N->dump(&DAG);
1524  dbgs() << "\n";
1525 #endif
1526  report_fatal_error("Do not know how to split this operator's "
1527  "operand!\n");
1528 
1529  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1530  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1531  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1532  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1533  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1534  case ISD::TRUNCATE:
1535  Res = SplitVecOp_TruncateHelper(N);
1536  break;
1537  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1538  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1539  case ISD::STORE:
1540  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1541  break;
1542  case ISD::MSTORE:
1543  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1544  break;
1545  case ISD::MSCATTER:
1546  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1547  break;
1548  case ISD::MGATHER:
1549  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1550  break;
1551  case ISD::VSELECT:
1552  Res = SplitVecOp_VSELECT(N, OpNo);
1553  break;
1554  case ISD::FP_TO_SINT:
1555  case ISD::FP_TO_UINT:
1556  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1557  Res = SplitVecOp_TruncateHelper(N);
1558  else
1559  Res = SplitVecOp_UnaryOp(N);
1560  break;
1561  case ISD::SINT_TO_FP:
1562  case ISD::UINT_TO_FP:
1563  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1564  Res = SplitVecOp_TruncateHelper(N);
1565  else
1566  Res = SplitVecOp_UnaryOp(N);
1567  break;
1568  case ISD::CTTZ:
1569  case ISD::CTLZ:
1570  case ISD::CTPOP:
1571  case ISD::FP_EXTEND:
1572  case ISD::SIGN_EXTEND:
1573  case ISD::ZERO_EXTEND:
1574  case ISD::ANY_EXTEND:
1575  case ISD::FTRUNC:
1576  case ISD::FCANONICALIZE:
1577  Res = SplitVecOp_UnaryOp(N);
1578  break;
1579 
1583  Res = SplitVecOp_ExtVecInRegOp(N);
1584  break;
1585 
1586  case ISD::VECREDUCE_FADD:
1587  case ISD::VECREDUCE_FMUL:
1588  case ISD::VECREDUCE_ADD:
1589  case ISD::VECREDUCE_MUL:
1590  case ISD::VECREDUCE_AND:
1591  case ISD::VECREDUCE_OR:
1592  case ISD::VECREDUCE_XOR:
1593  case ISD::VECREDUCE_SMAX:
1594  case ISD::VECREDUCE_SMIN:
1595  case ISD::VECREDUCE_UMAX:
1596  case ISD::VECREDUCE_UMIN:
1597  case ISD::VECREDUCE_FMAX:
1598  case ISD::VECREDUCE_FMIN:
1599  Res = SplitVecOp_VECREDUCE(N, OpNo);
1600  break;
1601  }
1602  }
1603 
1604  // If the result is null, the sub-method took care of registering results etc.
1605  if (!Res.getNode()) return false;
1606 
1607  // If the result is N, the sub-method updated N in place. Tell the legalizer
1608  // core about this.
1609  if (Res.getNode() == N)
1610  return true;
1611 
1612  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1613  "Invalid operand expansion");
1614 
1615  ReplaceValueWith(SDValue(N, 0), Res);
1616  return false;
1617 }
1618 
1619 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1620  // The only possibility for an illegal operand is the mask, since result type
1621  // legalization would have handled this node already otherwise.
1622  assert(OpNo == 0 && "Illegal operand must be mask");
1623 
1624  SDValue Mask = N->getOperand(0);
1625  SDValue Src0 = N->getOperand(1);
1626  SDValue Src1 = N->getOperand(2);
1627  EVT Src0VT = Src0.getValueType();
1628  SDLoc DL(N);
1629  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1630 
1631  SDValue Lo, Hi;
1632  GetSplitVector(N->getOperand(0), Lo, Hi);
1633  assert(Lo.getValueType() == Hi.getValueType() &&
1634  "Lo and Hi have differing types");
1635 
1636  EVT LoOpVT, HiOpVT;
1637  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1638  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1639 
1640  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1641  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1642  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1643  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1644 
1645  SDValue LoSelect =
1646  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1647  SDValue HiSelect =
1648  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1649 
1650  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1651 }
1652 
1653 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
1654  EVT ResVT = N->getValueType(0);
1655  SDValue Lo, Hi;
1656  SDLoc dl(N);
1657 
1658  SDValue VecOp = N->getOperand(OpNo);
1659  EVT VecVT = VecOp.getValueType();
1660  assert(VecVT.isVector() && "Can only split reduce vector operand");
1661  GetSplitVector(VecOp, Lo, Hi);
1662  EVT LoOpVT, HiOpVT;
1663  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
1664 
1665  bool NoNaN = N->getFlags().hasNoNaNs();
1666  unsigned CombineOpc = 0;
1667  switch (N->getOpcode()) {
1668  case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
1669  case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
1670  case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
1671  case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
1672  case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
1673  case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
1674  case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
1675  case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
1676  case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
1677  case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
1678  case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
1679  case ISD::VECREDUCE_FMAX:
1680  CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
1681  break;
1682  case ISD::VECREDUCE_FMIN:
1683  CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
1684  break;
1685  default:
1686  llvm_unreachable("Unexpected reduce ISD node");
1687  }
1688 
1689  // Use the appropriate scalar instruction on the split subvectors before
1690  // reducing the now partially reduced smaller vector.
1691  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
1692  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
1693 }
1694 
1695 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1696  // The result has a legal vector type, but the input needs splitting.
1697  EVT ResVT = N->getValueType(0);
1698  SDValue Lo, Hi;
1699  SDLoc dl(N);
1700  GetSplitVector(N->getOperand(0), Lo, Hi);
1701  EVT InVT = Lo.getValueType();
1702 
1703  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1704  InVT.getVectorNumElements());
1705 
1706  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1707  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1708 
1709  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1710 }
1711 
1712 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1713  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1714  // end up being split all the way down to individual components. Convert the
1715  // split pieces into integers and reassemble.
1716  SDValue Lo, Hi;
1717  GetSplitVector(N->getOperand(0), Lo, Hi);
1718  Lo = BitConvertToInteger(Lo);
1719  Hi = BitConvertToInteger(Hi);
1720 
1721  if (DAG.getDataLayout().isBigEndian())
1722  std::swap(Lo, Hi);
1723 
1724  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1725  JoinIntegers(Lo, Hi));
1726 }
1727 
1728 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1729  // We know that the extracted result type is legal.
1730  EVT SubVT = N->getValueType(0);
1731  SDValue Idx = N->getOperand(1);
1732  SDLoc dl(N);
1733  SDValue Lo, Hi;
1734  GetSplitVector(N->getOperand(0), Lo, Hi);
1735 
1736  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1737  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1738 
1739  if (IdxVal < LoElts) {
1740  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1741  "Extracted subvector crosses vector split!");
1742  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1743  } else {
1744  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1745  DAG.getConstant(IdxVal - LoElts, dl,
1746  Idx.getValueType()));
1747  }
1748 }
1749 
1750 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1751  SDValue Vec = N->getOperand(0);
1752  SDValue Idx = N->getOperand(1);
1753  EVT VecVT = Vec.getValueType();
1754 
1755  if (isa<ConstantSDNode>(Idx)) {
1756  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1757  assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1758 
1759  SDValue Lo, Hi;
1760  GetSplitVector(Vec, Lo, Hi);
1761 
1762  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1763 
1764  if (IdxVal < LoElts)
1765  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1766  return SDValue(DAG.UpdateNodeOperands(N, Hi,
1767  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
1768  Idx.getValueType())), 0);
1769  }
1770 
1771  // See if the target wants to custom expand this node.
1772  if (CustomLowerNode(N, N->getValueType(0), true))
1773  return SDValue();
1774 
1775  // Make the vector elements byte-addressable if they aren't already.
1776  SDLoc dl(N);
1777  EVT EltVT = VecVT.getVectorElementType();
1778  if (VecVT.getScalarSizeInBits() < 8) {
1779  EltVT = MVT::i8;
1780  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1781  VecVT.getVectorNumElements());
1782  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1783  }
1784 
1785  // Store the vector to the stack.
1786  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1787  auto &MF = DAG.getMachineFunction();
1788  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1789  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1790  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1791 
1792  // Load back the required element.
1793  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1794  return DAG.getExtLoad(
1795  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1797 }
1798 
1799 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
1800  SDValue Lo, Hi;
1801 
1802  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
1803  // splitting the result has the same effect as splitting the input operand.
1804  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1805 
1806  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
1807 }
1808 
1809 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
1810  unsigned OpNo) {
1811  EVT LoVT, HiVT;
1812  SDLoc dl(MGT);
1813  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1814 
1815  SDValue Ch = MGT->getChain();
1816  SDValue Ptr = MGT->getBasePtr();
1817  SDValue Index = MGT->getIndex();
1818  SDValue Scale = MGT->getScale();
1819  SDValue Mask = MGT->getMask();
1820  SDValue Src0 = MGT->getValue();
1821  unsigned Alignment = MGT->getOriginalAlignment();
1822 
1823  SDValue MaskLo, MaskHi;
1824  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1825  // Split Mask operand
1826  GetSplitVector(Mask, MaskLo, MaskHi);
1827  else
1828  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1829 
1830  EVT MemoryVT = MGT->getMemoryVT();
1831  EVT LoMemVT, HiMemVT;
1832  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1833 
1834  SDValue Src0Lo, Src0Hi;
1835  if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector)
1836  GetSplitVector(Src0, Src0Lo, Src0Hi);
1837  else
1838  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1839 
1840  SDValue IndexHi, IndexLo;
1841  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1842  GetSplitVector(Index, IndexLo, IndexHi);
1843  else
1844  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1845 
1847  getMachineMemOperand(MGT->getPointerInfo(),
1849  Alignment, MGT->getAAInfo(), MGT->getRanges());
1850 
1851  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
1852  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
1853  OpsLo, MMO);
1854 
1855  MMO = DAG.getMachineFunction().
1856  getMachineMemOperand(MGT->getPointerInfo(),
1858  Alignment, MGT->getAAInfo(),
1859  MGT->getRanges());
1860 
1861  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
1862  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
1863  OpsHi, MMO);
1864 
1865  // Build a factor node to remember that this load is independent of the
1866  // other one.
1867  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1868  Hi.getValue(1));
1869 
1870  // Legalize the chain result - switch anything that used the old chain to
1871  // use the new one.
1872  ReplaceValueWith(SDValue(MGT, 1), Ch);
1873 
1874  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
1875  Hi);
1876  ReplaceValueWith(SDValue(MGT, 0), Res);
1877  return SDValue();
1878 }
1879 
1880 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
1881  unsigned OpNo) {
1882  SDValue Ch = N->getChain();
1883  SDValue Ptr = N->getBasePtr();
1884  SDValue Mask = N->getMask();
1885  SDValue Data = N->getValue();
1886  EVT MemoryVT = N->getMemoryVT();
1887  unsigned Alignment = N->getOriginalAlignment();
1888  SDLoc DL(N);
1889 
1890  EVT LoMemVT, HiMemVT;
1891  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1892 
1893  SDValue DataLo, DataHi;
1894  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1895  // Split Data operand
1896  GetSplitVector(Data, DataLo, DataHi);
1897  else
1898  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1899 
1900  SDValue MaskLo, MaskHi;
1901  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1902  // Split Mask operand
1903  GetSplitVector(Mask, MaskLo, MaskHi);
1904  else
1905  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1906 
1907  // if Alignment is equal to the vector size,
1908  // take the half of it for the second part
1909  unsigned SecondHalfAlignment =
1910  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
1911  Alignment/2 : Alignment;
1912 
1913  SDValue Lo, Hi;
1915  getMachineMemOperand(N->getPointerInfo(),
1917  Alignment, N->getAAInfo(), N->getRanges());
1918 
1919  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
1920  N->isTruncatingStore(),
1921  N->isCompressingStore());
1922 
1923  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
1924  N->isCompressingStore());
1925  unsigned HiOffset = LoMemVT.getStoreSize();
1926 
1929  HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
1930  N->getRanges());
1931 
1932  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
1934 
1935  // Build a factor node to remember that this store is independent of the
1936  // other one.
1937  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1938 }
1939 
1940 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
1941  unsigned OpNo) {
1942  SDValue Ch = N->getChain();
1943  SDValue Ptr = N->getBasePtr();
1944  SDValue Mask = N->getMask();
1945  SDValue Index = N->getIndex();
1946  SDValue Scale = N->getScale();
1947  SDValue Data = N->getValue();
1948  EVT MemoryVT = N->getMemoryVT();
1949  unsigned Alignment = N->getOriginalAlignment();
1950  SDLoc DL(N);
1951 
1952  // Split all operands
1953  EVT LoMemVT, HiMemVT;
1954  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1955 
1956  SDValue DataLo, DataHi;
1957  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
1958  // Split Data operand
1959  GetSplitVector(Data, DataLo, DataHi);
1960  else
1961  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
1962 
1963  SDValue MaskLo, MaskHi;
1964  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1965  // Split Mask operand
1966  GetSplitVector(Mask, MaskLo, MaskHi);
1967  else
1968  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1969 
1970  SDValue IndexHi, IndexLo;
1971  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1972  GetSplitVector(Index, IndexLo, IndexHi);
1973  else
1974  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
1975 
1976  SDValue Lo;
1978  getMachineMemOperand(N->getPointerInfo(),
1980  Alignment, N->getAAInfo(), N->getRanges());
1981 
1982  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
1983  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
1984  DL, OpsLo, MMO);
1985 
1986  MMO = DAG.getMachineFunction().
1987  getMachineMemOperand(N->getPointerInfo(),
1989  Alignment, N->getAAInfo(), N->getRanges());
1990 
1991  // The order of the Scatter operation after split is well defined. The "Hi"
1992  // part comes after the "Lo". So these two operations should be chained one
1993  // after another.
1994  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
1995  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
1996  DL, OpsHi, MMO);
1997 }
1998 
1999 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2000  assert(N->isUnindexed() && "Indexed store of vector?");
2001  assert(OpNo == 1 && "Can only split the stored value");
2002  SDLoc DL(N);
2003 
2004  bool isTruncating = N->isTruncatingStore();
2005  SDValue Ch = N->getChain();
2006  SDValue Ptr = N->getBasePtr();
2007  EVT MemoryVT = N->getMemoryVT();
2008  unsigned Alignment = N->getOriginalAlignment();
2009  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2010  AAMDNodes AAInfo = N->getAAInfo();
2011  SDValue Lo, Hi;
2012  GetSplitVector(N->getOperand(1), Lo, Hi);
2013 
2014  EVT LoMemVT, HiMemVT;
2015  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2016 
2017  // Scalarize if the split halves are not byte-sized.
2018  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2019  return TLI.scalarizeVectorStore(N, DAG);
2020 
2021  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2022 
2023  if (isTruncating)
2024  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2025  Alignment, MMOFlags, AAInfo);
2026  else
2027  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2028  AAInfo);
2029 
2030  // Increment the pointer to the other half.
2031  Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2032 
2033  if (isTruncating)
2034  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2035  N->getPointerInfo().getWithOffset(IncrementSize),
2036  HiMemVT, Alignment, MMOFlags, AAInfo);
2037  else
2038  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2039  N->getPointerInfo().getWithOffset(IncrementSize),
2040  Alignment, MMOFlags, AAInfo);
2041 
2042  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2043 }
2044 
2045 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2046  SDLoc DL(N);
2047 
2048  // The input operands all must have the same type, and we know the result
2049  // type is valid. Convert this to a buildvector which extracts all the
2050  // input elements.
2051  // TODO: If the input elements are power-two vectors, we could convert this to
2052  // a new CONCAT_VECTORS node with elements that are half-wide.
2054  EVT EltVT = N->getValueType(0).getVectorElementType();
2055  for (const SDValue &Op : N->op_values()) {
2056  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2057  i != e; ++i) {
2058  Elts.push_back(DAG.getNode(
2059  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2060  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2061  }
2062  }
2063 
2064  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2065 }
2066 
2067 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2068  // The result type is legal, but the input type is illegal. If splitting
2069  // ends up with the result type of each half still being legal, just
2070  // do that. If, however, that would result in an illegal result type,
2071  // we can try to get more clever with power-two vectors. Specifically,
2072  // split the input type, but also widen the result element size, then
2073  // concatenate the halves and truncate again. For example, consider a target
2074  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2075  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2076  // %inlo = v4i32 extract_subvector %in, 0
2077  // %inhi = v4i32 extract_subvector %in, 4
2078  // %lo16 = v4i16 trunc v4i32 %inlo
2079  // %hi16 = v4i16 trunc v4i32 %inhi
2080  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2081  // %res = v8i8 trunc v8i16 %in16
2082  //
2083  // Without this transform, the original truncate would end up being
2084  // scalarized, which is pretty much always a last resort.
2085  SDValue InVec = N->getOperand(0);
2086  EVT InVT = InVec->getValueType(0);
2087  EVT OutVT = N->getValueType(0);
2088  unsigned NumElements = OutVT.getVectorNumElements();
2089  bool IsFloat = OutVT.isFloatingPoint();
2090 
2091  // Widening should have already made sure this is a power-two vector
2092  // if we're trying to split it at all. assert() that's true, just in case.
2093  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2094 
2095  unsigned InElementSize = InVT.getScalarSizeInBits();
2096  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2097 
2098  // If the input elements are only 1/2 the width of the result elements,
2099  // just use the normal splitting. Our trick only work if there's room
2100  // to split more than once.
2101  if (InElementSize <= OutElementSize * 2)
2102  return SplitVecOp_UnaryOp(N);
2103  SDLoc DL(N);
2104 
2105  // Get the split input vector.
2106  SDValue InLoVec, InHiVec;
2107  GetSplitVector(InVec, InLoVec, InHiVec);
2108  // Truncate them to 1/2 the element size.
2109  EVT HalfElementVT = IsFloat ?
2110  EVT::getFloatingPointVT(InElementSize/2) :
2111  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2112  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2113  NumElements/2);
2114  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2115  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2116  // Concatenate them to get the full intermediate truncation result.
2117  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2118  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2119  HalfHi);
2120  // Now finish up by truncating all the way down to the original result
2121  // type. This should normally be something that ends up being legal directly,
2122  // but in theory if a target has very wide vectors and an annoyingly
2123  // restricted set of legal types, this split can chain to build things up.
2124  return IsFloat
2125  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2126  DAG.getTargetConstant(
2127  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2128  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2129 }
2130 
2131 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2132  assert(N->getValueType(0).isVector() &&
2133  N->getOperand(0).getValueType().isVector() &&
2134  "Operand types must be vectors");
2135  // The result has a legal vector type, but the input needs splitting.
2136  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2137  SDLoc DL(N);
2138  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2139  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2140  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2141  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2142  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2143 
2144  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2145  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2146  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2147  return PromoteTargetBoolean(Con, N->getValueType(0));
2148 }
2149 
2150 
2151 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2152  // The result has a legal vector type, but the input needs splitting.
2153  EVT ResVT = N->getValueType(0);
2154  SDValue Lo, Hi;
2155  SDLoc DL(N);
2156  GetSplitVector(N->getOperand(0), Lo, Hi);
2157  EVT InVT = Lo.getValueType();
2158 
2159  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2160  InVT.getVectorNumElements());
2161 
2162  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2163  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2164 
2165  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2166 }
2167 
2168 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2169  // The result (and the first input) has a legal vector type, but the second
2170  // input needs splitting.
2171  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2172 }
2173 
2174 
2175 //===----------------------------------------------------------------------===//
2176 // Result Vector Widening
2177 //===----------------------------------------------------------------------===//
2178 
2179 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2180  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2181  dbgs() << "\n");
2182 
2183  // See if the target wants to custom widen this node.
2184  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2185  return;
2186 
2187  SDValue Res = SDValue();
2188  switch (N->getOpcode()) {
2189  default:
2190 #ifndef NDEBUG
2191  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2192  N->dump(&DAG);
2193  dbgs() << "\n";
2194 #endif
2195  llvm_unreachable("Do not know how to widen the result of this operator!");
2196 
2197  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2198  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2199  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2200  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2201  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2202  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2203  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2204  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2205  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2206  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2207  case ISD::VSELECT:
2208  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2209  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2210  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2211  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2212  case ISD::VECTOR_SHUFFLE:
2213  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2214  break;
2215  case ISD::MLOAD:
2216  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2217  break;
2218  case ISD::MGATHER:
2219  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2220  break;
2221 
2222  case ISD::ADD:
2223  case ISD::AND:
2224  case ISD::MUL:
2225  case ISD::MULHS:
2226  case ISD::MULHU:
2227  case ISD::OR:
2228  case ISD::SUB:
2229  case ISD::XOR:
2230  case ISD::FMINNUM:
2231  case ISD::FMAXNUM:
2232  case ISD::FMINNAN:
2233  case ISD::FMAXNAN:
2234  case ISD::SMIN:
2235  case ISD::SMAX:
2236  case ISD::UMIN:
2237  case ISD::UMAX:
2238  Res = WidenVecRes_Binary(N);
2239  break;
2240 
2241  case ISD::FADD:
2242  case ISD::FMUL:
2243  case ISD::FPOW:
2244  case ISD::FSUB:
2245  case ISD::FDIV:
2246  case ISD::FREM:
2247  case ISD::SDIV:
2248  case ISD::UDIV:
2249  case ISD::SREM:
2250  case ISD::UREM:
2251  Res = WidenVecRes_BinaryCanTrap(N);
2252  break;
2253 
2254  case ISD::FCOPYSIGN:
2255  Res = WidenVecRes_FCOPYSIGN(N);
2256  break;
2257 
2258  case ISD::FPOWI:
2259  Res = WidenVecRes_POWI(N);
2260  break;
2261 
2262  case ISD::SHL:
2263  case ISD::SRA:
2264  case ISD::SRL:
2265  Res = WidenVecRes_Shift(N);
2266  break;
2267 
2271  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2272  break;
2273 
2274  case ISD::ANY_EXTEND:
2275  case ISD::FP_EXTEND:
2276  case ISD::FP_ROUND:
2277  case ISD::FP_TO_SINT:
2278  case ISD::FP_TO_UINT:
2279  case ISD::SIGN_EXTEND:
2280  case ISD::SINT_TO_FP:
2281  case ISD::TRUNCATE:
2282  case ISD::UINT_TO_FP:
2283  case ISD::ZERO_EXTEND:
2284  Res = WidenVecRes_Convert(N);
2285  break;
2286 
2287  case ISD::BITREVERSE:
2288  case ISD::BSWAP:
2289  case ISD::CTLZ:
2290  case ISD::CTPOP:
2291  case ISD::CTTZ:
2292  case ISD::FABS:
2293  case ISD::FCEIL:
2294  case ISD::FCOS:
2295  case ISD::FEXP:
2296  case ISD::FEXP2:
2297  case ISD::FFLOOR:
2298  case ISD::FLOG:
2299  case ISD::FLOG10:
2300  case ISD::FLOG2:
2301  case ISD::FNEARBYINT:
2302  case ISD::FNEG:
2303  case ISD::FRINT:
2304  case ISD::FROUND:
2305  case ISD::FSIN:
2306  case ISD::FSQRT:
2307  case ISD::FTRUNC:
2308  Res = WidenVecRes_Unary(N);
2309  break;
2310  case ISD::FMA:
2311  Res = WidenVecRes_Ternary(N);
2312  break;
2313  }
2314 
2315  // If Res is null, the sub-method took care of registering the result.
2316  if (Res.getNode())
2317  SetWidenedVector(SDValue(N, ResNo), Res);
2318 }
2319 
2320 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2321  // Ternary op widening.
2322  SDLoc dl(N);
2323  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2324  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2325  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2326  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2327  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2328 }
2329 
2330 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2331  // Binary op widening.
2332  SDLoc dl(N);
2333  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2334  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2335  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2336  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2337 }
2338 
2339 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2340  // Binary op widening for operations that can trap.
2341  unsigned Opcode = N->getOpcode();
2342  SDLoc dl(N);
2343  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2344  EVT WidenEltVT = WidenVT.getVectorElementType();
2345  EVT VT = WidenVT;
2346  unsigned NumElts = VT.getVectorNumElements();
2347  const SDNodeFlags Flags = N->getFlags();
2348  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2349  NumElts = NumElts / 2;
2350  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2351  }
2352 
2353  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2354  // Operation doesn't trap so just widen as normal.
2355  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2356  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2357  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2358  }
2359 
2360  // No legal vector version so unroll the vector operation and then widen.
2361  if (NumElts == 1)
2362  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2363 
2364  // Since the operation can trap, apply operation on the original vector.
2365  EVT MaxVT = VT;
2366  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2367  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2368  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2369 
2370  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2371  unsigned ConcatEnd = 0; // Current ConcatOps index.
2372  int Idx = 0; // Current Idx into input vectors.
2373 
2374  // NumElts := greatest legal vector size (at most WidenVT)
2375  // while (orig. vector has unhandled elements) {
2376  // take munches of size NumElts from the beginning and add to ConcatOps
2377  // NumElts := next smaller supported vector size or 1
2378  // }
2379  while (CurNumElts != 0) {
2380  while (CurNumElts >= NumElts) {
2381  SDValue EOp1 = DAG.getNode(
2382  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2383  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2384  SDValue EOp2 = DAG.getNode(
2385  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2386  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2387  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2388  Idx += NumElts;
2389  CurNumElts -= NumElts;
2390  }
2391  do {
2392  NumElts = NumElts / 2;
2393  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2394  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2395 
2396  if (NumElts == 1) {
2397  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2398  SDValue EOp1 = DAG.getNode(
2399  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2400  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2401  SDValue EOp2 = DAG.getNode(
2402  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2403  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2404  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2405  EOp1, EOp2, Flags);
2406  }
2407  CurNumElts = 0;
2408  }
2409  }
2410 
2411  // Check to see if we have a single operation with the widen type.
2412  if (ConcatEnd == 1) {
2413  VT = ConcatOps[0].getValueType();
2414  if (VT == WidenVT)
2415  return ConcatOps[0];
2416  }
2417 
2418  // while (Some element of ConcatOps is not of type MaxVT) {
2419  // From the end of ConcatOps, collect elements of the same type and put
2420  // them into an op of the next larger supported type
2421  // }
2422  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2423  Idx = ConcatEnd - 1;
2424  VT = ConcatOps[Idx--].getValueType();
2425  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2426  Idx--;
2427 
2428  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2429  EVT NextVT;
2430  do {
2431  NextSize *= 2;
2432  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2433  } while (!TLI.isTypeLegal(NextVT));
2434 
2435  if (!VT.isVector()) {
2436  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2437  SDValue VecOp = DAG.getUNDEF(NextVT);
2438  unsigned NumToInsert = ConcatEnd - Idx - 1;
2439  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2440  VecOp = DAG.getNode(
2441  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2442  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2443  }
2444  ConcatOps[Idx+1] = VecOp;
2445  ConcatEnd = Idx + 2;
2446  } else {
2447  // Vector type, create a CONCAT_VECTORS of type NextVT
2448  SDValue undefVec = DAG.getUNDEF(VT);
2449  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2450  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2451  unsigned RealVals = ConcatEnd - Idx - 1;
2452  unsigned SubConcatEnd = 0;
2453  unsigned SubConcatIdx = Idx + 1;
2454  while (SubConcatEnd < RealVals)
2455  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2456  while (SubConcatEnd < OpsToConcat)
2457  SubConcatOps[SubConcatEnd++] = undefVec;
2458  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2459  NextVT, SubConcatOps);
2460  ConcatEnd = SubConcatIdx + 1;
2461  }
2462  }
2463 
2464  // Check to see if we have a single operation with the widen type.
2465  if (ConcatEnd == 1) {
2466  VT = ConcatOps[0].getValueType();
2467  if (VT == WidenVT)
2468  return ConcatOps[0];
2469  }
2470 
2471  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2472  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2473  if (NumOps != ConcatEnd ) {
2474  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2475  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2476  ConcatOps[j] = UndefVal;
2477  }
2478  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2479  makeArrayRef(ConcatOps.data(), NumOps));
2480 }
2481 
2482 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
2483  SDValue InOp = N->getOperand(0);
2484  SDLoc DL(N);
2485 
2486  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2487  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2488 
2489  EVT InVT = InOp.getValueType();
2490  EVT InEltVT = InVT.getVectorElementType();
2491  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2492 
2493  unsigned Opcode = N->getOpcode();
2494  unsigned InVTNumElts = InVT.getVectorNumElements();
2495  const SDNodeFlags Flags = N->getFlags();
2496  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2497  InOp = GetWidenedVector(N->getOperand(0));
2498  InVT = InOp.getValueType();
2499  InVTNumElts = InVT.getVectorNumElements();
2500  if (InVTNumElts == WidenNumElts) {
2501  if (N->getNumOperands() == 1)
2502  return DAG.getNode(Opcode, DL, WidenVT, InOp);
2503  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
2504  }
2505  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
2506  // If both input and result vector types are of same width, extend
2507  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
2508  // accepts fewer elements in the result than in the input.
2509  if (Opcode == ISD::SIGN_EXTEND)
2510  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2511  if (Opcode == ISD::ZERO_EXTEND)
2512  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2513  }
2514  }
2515 
2516  if (TLI.isTypeLegal(InWidenVT)) {
2517  // Because the result and the input are different vector types, widening
2518  // the result could create a legal type but widening the input might make
2519  // it an illegal type that might lead to repeatedly splitting the input
2520  // and then widening it. To avoid this, we widen the input only if
2521  // it results in a legal type.
2522  if (WidenNumElts % InVTNumElts == 0) {
2523  // Widen the input and call convert on the widened input vector.
2524  unsigned NumConcat = WidenNumElts/InVTNumElts;
2525  SmallVector<SDValue, 16> Ops(NumConcat);
2526  Ops[0] = InOp;
2527  SDValue UndefVal = DAG.getUNDEF(InVT);
2528  for (unsigned i = 1; i != NumConcat; ++i)
2529  Ops[i] = UndefVal;
2530  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
2531  if (N->getNumOperands() == 1)
2532  return DAG.getNode(Opcode, DL, WidenVT, InVec);
2533  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
2534  }
2535 
2536  if (InVTNumElts % WidenNumElts == 0) {
2537  SDValue InVal = DAG.getNode(
2538  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
2539  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2540  // Extract the input and convert the shorten input vector.
2541  if (N->getNumOperands() == 1)
2542  return DAG.getNode(Opcode, DL, WidenVT, InVal);
2543  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
2544  }
2545  }
2546 
2547  // Otherwise unroll into some nasty scalar code and rebuild the vector.
2548  SmallVector<SDValue, 16> Ops(WidenNumElts);
2549  EVT EltVT = WidenVT.getVectorElementType();
2550  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2551  unsigned i;
2552  for (i=0; i < MinElts; ++i) {
2553  SDValue Val = DAG.getNode(
2554  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
2555  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2556  if (N->getNumOperands() == 1)
2557  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
2558  else
2559  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
2560  }
2561 
2562  SDValue UndefVal = DAG.getUNDEF(EltVT);
2563  for (; i < WidenNumElts; ++i)
2564  Ops[i] = UndefVal;
2565 
2566  return DAG.getBuildVector(WidenVT, DL, Ops);
2567 }
2568 
2569 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
2570  unsigned Opcode = N->getOpcode();
2571  SDValue InOp = N->getOperand(0);
2572  SDLoc DL(N);
2573 
2574  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2575  EVT WidenSVT = WidenVT.getVectorElementType();
2576  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2577 
2578  EVT InVT = InOp.getValueType();
2579  EVT InSVT = InVT.getVectorElementType();
2580  unsigned InVTNumElts = InVT.getVectorNumElements();
2581 
2582  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2583  InOp = GetWidenedVector(InOp);
2584  InVT = InOp.getValueType();
2585  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
2586  switch (Opcode) {
2588  return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
2590  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2592  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2593  }
2594  }
2595  }
2596 
2597  // Unroll, extend the scalars and rebuild the vector.
2599  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
2600  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
2601  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2602  switch (Opcode) {
2604  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
2605  break;
2607  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
2608  break;
2610  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
2611  break;
2612  default:
2613  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
2614  }
2615  Ops.push_back(Val);
2616  }
2617 
2618  while (Ops.size() != WidenNumElts)
2619  Ops.push_back(DAG.getUNDEF(WidenSVT));
2620 
2621  return DAG.getBuildVector(WidenVT, DL, Ops);
2622 }
2623 
2624 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
2625  // If this is an FCOPYSIGN with same input types, we can treat it as a
2626  // normal (can trap) binary op.
2627  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
2628  return WidenVecRes_BinaryCanTrap(N);
2629 
2630  // If the types are different, fall back to unrolling.
2631  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2632  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2633 }
2634 
2635 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
2636  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2637  SDValue InOp = GetWidenedVector(N->getOperand(0));
2638  SDValue ShOp = N->getOperand(1);
2639  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2640 }
2641 
2642 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
2643  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2644  SDValue InOp = GetWidenedVector(N->getOperand(0));
2645  SDValue ShOp = N->getOperand(1);
2646 
2647  EVT ShVT = ShOp.getValueType();
2648  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
2649  ShOp = GetWidenedVector(ShOp);
2650  ShVT = ShOp.getValueType();
2651  }
2652  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
2653  ShVT.getVectorElementType(),
2654  WidenVT.getVectorNumElements());
2655  if (ShVT != ShWidenVT)
2656  ShOp = ModifyToType(ShOp, ShWidenVT);
2657 
2658  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2659 }
2660 
2661 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
2662  // Unary op widening.
2663  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2664  SDValue InOp = GetWidenedVector(N->getOperand(0));
2665  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
2666 }
2667 
2668 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
2669  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2670  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
2671  cast<VTSDNode>(N->getOperand(1))->getVT()
2672  .getVectorElementType(),
2673  WidenVT.getVectorNumElements());
2674  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
2675  return DAG.getNode(N->getOpcode(), SDLoc(N),
2676  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
2677 }
2678 
2679 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
2680  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2681  return GetWidenedVector(WidenVec);
2682 }
2683 
2684 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
2685  SDValue InOp = N->getOperand(0);
2686  EVT InVT = InOp.getValueType();
2687  EVT VT = N->getValueType(0);
2688  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2689  SDLoc dl(N);
2690 
2691  switch (getTypeAction(InVT)) {
2693  break;
2695  // If the incoming type is a vector that is being promoted, then
2696  // we know that the elements are arranged differently and that we
2697  // must perform the conversion using a stack slot.
2698  if (InVT.isVector())
2699  break;
2700 
2701  // If the InOp is promoted to the same size, convert it. Otherwise,
2702  // fall out of the switch and widen the promoted input.
2703  InOp = GetPromotedInteger(InOp);
2704  InVT = InOp.getValueType();
2705  if (WidenVT.bitsEq(InVT))
2706  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2707  break;
2714  break;
2716  // If the InOp is widened to the same size, convert it. Otherwise, fall
2717  // out of the switch and widen the widened input.
2718  InOp = GetWidenedVector(InOp);
2719  InVT = InOp.getValueType();
2720  if (WidenVT.bitsEq(InVT))
2721  // The input widens to the same size. Convert to the widen value.
2722  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2723  break;
2724  }
2725 
2726  unsigned WidenSize = WidenVT.getSizeInBits();
2727  unsigned InSize = InVT.getSizeInBits();
2728  // x86mmx is not an acceptable vector element type, so don't try.
2729  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
2730  // Determine new input vector type. The new input vector type will use
2731  // the same element type (if its a vector) or use the input type as a
2732  // vector. It is the same size as the type to widen to.
2733  EVT NewInVT;
2734  unsigned NewNumElts = WidenSize / InSize;
2735  if (InVT.isVector()) {
2736  EVT InEltVT = InVT.getVectorElementType();
2737  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
2738  WidenSize / InEltVT.getSizeInBits());
2739  } else {
2740  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
2741  }
2742 
2743  if (TLI.isTypeLegal(NewInVT)) {
2744  // Because the result and the input are different vector types, widening
2745  // the result could create a legal type but widening the input might make
2746  // it an illegal type that might lead to repeatedly splitting the input
2747  // and then widening it. To avoid this, we widen the input only if
2748  // it results in a legal type.
2749  SmallVector<SDValue, 16> Ops(NewNumElts);
2750  SDValue UndefVal = DAG.getUNDEF(InVT);
2751  Ops[0] = InOp;
2752  for (unsigned i = 1; i < NewNumElts; ++i)
2753  Ops[i] = UndefVal;
2754 
2755  SDValue NewVec;
2756  if (InVT.isVector())
2757  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
2758  else
2759  NewVec = DAG.getBuildVector(NewInVT, dl, Ops);
2760  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
2761  }
2762  }
2763 
2764  return CreateStackStoreLoad(InOp, WidenVT);
2765 }
2766 
2767 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
2768  SDLoc dl(N);
2769  // Build a vector with undefined for the new nodes.
2770  EVT VT = N->getValueType(0);
2771 
2772  // Integer BUILD_VECTOR operands may be larger than the node's vector element
2773  // type. The UNDEFs need to have the same type as the existing operands.
2774  EVT EltVT = N->getOperand(0).getValueType();
2775  unsigned NumElts = VT.getVectorNumElements();
2776 
2777  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2778  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2779 
2780  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
2781  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
2782  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
2783 
2784  return DAG.getBuildVector(WidenVT, dl, NewOps);
2785 }
2786 
2787 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
2788  EVT InVT = N->getOperand(0).getValueType();
2789  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2790  SDLoc dl(N);
2791  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2792  unsigned NumInElts = InVT.getVectorNumElements();
2793  unsigned NumOperands = N->getNumOperands();
2794 
2795  bool InputWidened = false; // Indicates we need to widen the input.
2796  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
2797  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
2798  // Add undef vectors to widen to correct length.
2799  unsigned NumConcat = WidenVT.getVectorNumElements() /
2800  InVT.getVectorNumElements();
2801  SDValue UndefVal = DAG.getUNDEF(InVT);
2802  SmallVector<SDValue, 16> Ops(NumConcat);
2803  for (unsigned i=0; i < NumOperands; ++i)
2804  Ops[i] = N->getOperand(i);
2805  for (unsigned i = NumOperands; i != NumConcat; ++i)
2806  Ops[i] = UndefVal;
2807  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
2808  }
2809  } else {
2810  InputWidened = true;
2811  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
2812  // The inputs and the result are widen to the same value.
2813  unsigned i;
2814  for (i=1; i < NumOperands; ++i)
2815  if (!N->getOperand(i).isUndef())
2816  break;
2817 
2818  if (i == NumOperands)
2819  // Everything but the first operand is an UNDEF so just return the
2820  // widened first operand.
2821  return GetWidenedVector(N->getOperand(0));
2822 
2823  if (NumOperands == 2) {
2824  // Replace concat of two operands with a shuffle.
2825  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
2826  for (unsigned i = 0; i < NumInElts; ++i) {
2827  MaskOps[i] = i;
2828  MaskOps[i + NumInElts] = i + WidenNumElts;
2829  }
2830  return DAG.getVectorShuffle(WidenVT, dl,
2831  GetWidenedVector(N->getOperand(0)),
2832  GetWidenedVector(N->getOperand(1)),
2833  MaskOps);
2834  }
2835  }
2836  }
2837 
2838  // Fall back to use extracts and build vector.
2839  EVT EltVT = WidenVT.getVectorElementType();
2840  SmallVector<SDValue, 16> Ops(WidenNumElts);
2841  unsigned Idx = 0;
2842  for (unsigned i=0; i < NumOperands; ++i) {
2843  SDValue InOp = N->getOperand(i);
2844  if (InputWidened)
2845  InOp = GetWidenedVector(InOp);
2846  for (unsigned j=0; j < NumInElts; ++j)
2847  Ops[Idx++] = DAG.getNode(
2848  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2849  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2850  }
2851  SDValue UndefVal = DAG.getUNDEF(EltVT);
2852  for (; Idx < WidenNumElts; ++Idx)
2853  Ops[Idx] = UndefVal;
2854  return DAG.getBuildVector(WidenVT, dl, Ops);
2855 }
2856 
2857 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
2858  EVT VT = N->getValueType(0);
2859  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2860  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2861  SDValue InOp = N->getOperand(0);
2862  SDValue Idx = N->getOperand(1);
2863  SDLoc dl(N);
2864 
2865  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2866  InOp = GetWidenedVector(InOp);
2867 
2868  EVT InVT = InOp.getValueType();
2869 
2870  // Check if we can just return the input vector after widening.
2871  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2872  if (IdxVal == 0 && InVT == WidenVT)
2873  return InOp;
2874 
2875  // Check if we can extract from the vector.
2876  unsigned InNumElts = InVT.getVectorNumElements();
2877  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2878  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
2879 
2880  // We could try widening the input to the right length but for now, extract
2881  // the original elements, fill the rest with undefs and build a vector.
2882  SmallVector<SDValue, 16> Ops(WidenNumElts);
2883  EVT EltVT = VT.getVectorElementType();
2884  unsigned NumElts = VT.getVectorNumElements();
2885  unsigned i;
2886  for (i=0; i < NumElts; ++i)
2887  Ops[i] =
2888  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2889  DAG.getConstant(IdxVal + i, dl,
2890  TLI.getVectorIdxTy(DAG.getDataLayout())));
2891 
2892  SDValue UndefVal = DAG.getUNDEF(EltVT);
2893  for (; i < WidenNumElts; ++i)
2894  Ops[i] = UndefVal;
2895  return DAG.getBuildVector(WidenVT, dl, Ops);
2896 }
2897 
2898 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
2899  SDValue InOp = GetWidenedVector(N->getOperand(0));
2900  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
2901  InOp.getValueType(), InOp,
2902  N->getOperand(1), N->getOperand(2));
2903 }
2904 
2905 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
2906  LoadSDNode *LD = cast<LoadSDNode>(N);
2907  ISD::LoadExtType ExtType = LD->getExtensionType();
2908 
2909  SDValue Result;
2910  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
2911  if (ExtType != ISD::NON_EXTLOAD)
2912  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2913  else
2914  Result = GenWidenVectorLoads(LdChain, LD);
2915 
2916  // If we generate a single load, we can use that for the chain. Otherwise,
2917  // build a factor node to remember the multiple loads are independent and
2918  // chain to that.
2919  SDValue NewChain;
2920  if (LdChain.size() == 1)
2921  NewChain = LdChain[0];
2922  else
2923  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
2924 
2925  // Modified the chain - switch anything that used the old chain to use
2926  // the new one.
2927  ReplaceValueWith(SDValue(N, 1), NewChain);
2928 
2929  return Result;
2930 }
2931 
2932 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
2933 
2934  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
2935  SDValue Mask = N->getMask();
2936  EVT MaskVT = Mask.getValueType();
2937  SDValue Src0 = GetWidenedVector(N->getSrc0());
2938  ISD::LoadExtType ExtType = N->getExtensionType();
2939  SDLoc dl(N);
2940 
2941  // The mask should be widened as well
2942  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
2943  MaskVT.getVectorElementType(),
2944  WidenVT.getVectorNumElements());
2945  Mask = ModifyToType(Mask, WideMaskVT, true);
2946 
2947  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
2948  Mask, Src0, N->getMemoryVT(),
2949  N->getMemOperand(), ExtType,
2950  N->isExpandingLoad());
2951  // Legalize the chain result - switch anything that used the old chain to
2952  // use the new one.
2953  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2954  return Res;
2955 }
2956 
2957 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
2958 
2959  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2960  SDValue Mask = N->getMask();
2961  EVT MaskVT = Mask.getValueType();
2962  SDValue Src0 = GetWidenedVector(N->getValue());
2963  SDValue Scale = N->getScale();
2964  unsigned NumElts = WideVT.getVectorNumElements();
2965  SDLoc dl(N);
2966 
2967  // The mask should be widened as well
2968  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
2969  MaskVT.getVectorElementType(),
2970  WideVT.getVectorNumElements());
2971  Mask = ModifyToType(Mask, WideMaskVT, true);
2972 
2973  // Widen the Index operand
2974  SDValue Index = N->getIndex();
2975  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
2976  Index.getValueType().getScalarType(),
2977  NumElts);
2978  Index = ModifyToType(Index, WideIndexVT);
2979  SDValue Ops[] = { N->getChain(), Src0, Mask, N->getBasePtr(), Index, Scale };
2980  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
2981  N->getMemoryVT(), dl, Ops,
2982  N->getMemOperand());
2983 
2984  // Legalize the chain result - switch anything that used the old chain to
2985  // use the new one.
2986  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2987  return Res;
2988 }
2989 
2990 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
2991  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2992  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
2993  WidenVT, N->getOperand(0));
2994 }
2995 
2996 // Return true if this is a node that could have two SETCCs as operands.
2997 static inline bool isLogicalMaskOp(unsigned Opcode) {
2998  switch (Opcode) {
2999  case ISD::AND:
3000  case ISD::OR:
3001  case ISD::XOR:
3002  return true;
3003  }
3004  return false;
3005 }
3006 
3007 // This is used just for the assert in convertMask(). Check that this either
3008 // a SETCC or a previously handled SETCC by convertMask().
3009 #ifndef NDEBUG
3010 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3011  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3012  N = N.getOperand(0);
3013  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3014  for (unsigned i = 1; i < N->getNumOperands(); ++i)
3015  if (!N->getOperand(i)->isUndef())
3016  return false;
3017  N = N.getOperand(0);
3018  }
3019 
3020  if (N.getOpcode() == ISD::TRUNCATE)
3021  N = N.getOperand(0);
3022  else if (N.getOpcode() == ISD::SIGN_EXTEND)
3023  N = N.getOperand(0);
3024 
3025  if (isLogicalMaskOp(N.getOpcode()))
3026  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3028 
3029  return (N.getOpcode() == ISD::SETCC ||
3031 }
3032 #endif
3033 
3034 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3035 // to ToMaskVT if needed with vector extension or truncation.
3036 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3037  EVT ToMaskVT) {
3038  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3039  // FIXME: This code seems to be too restrictive, we might consider
3040  // generalizing it or dropping it.
3041  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3042 
3043  // Make a new Mask node, with a legal result VT.
3045  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3046  Ops.push_back(InMask->getOperand(i));
3047  SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3048 
3049  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3050  // extend or truncate is needed.
3051  LLVMContext &Ctx = *DAG.getContext();
3052  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3053  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3054  if (MaskScalarBits < ToMaskScalBits) {
3055  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3056  MaskVT.getVectorNumElements());
3057  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3058  } else if (MaskScalarBits > ToMaskScalBits) {
3059  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3060  MaskVT.getVectorNumElements());
3061  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3062  }
3063 
3065  ToMaskVT.getScalarSizeInBits() &&
3066  "Mask should have the right element size by now.");
3067 
3068  // Adjust Mask to the right number of elements.
3069  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3070  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3071  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3072  SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3073  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3074  ZeroIdx);
3075  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3076  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3077  EVT SubVT = Mask->getValueType(0);
3078  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3079  SubOps[0] = Mask;
3080  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3081  }
3082 
3083  assert((Mask->getValueType(0) == ToMaskVT) &&
3084  "A mask of ToMaskVT should have been produced by now.");
3085 
3086  return Mask;
3087 }
3088 
3089 // Get the target mask VT, and widen if needed.
3090 EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
3091  assert(SetCC->getOpcode() == ISD::SETCC);
3092  LLVMContext &Ctx = *DAG.getContext();
3093  EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
3094  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3095  MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
3096  return MaskVT;
3097 }
3098 
3099 // This method tries to handle VSELECT and its mask by legalizing operands
3100 // (which may require widening) and if needed adjusting the mask vector type
3101 // to match that of the VSELECT. Without it, many cases end up with
3102 // scalarization of the SETCC, with many unnecessary instructions.
3103 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3104  LLVMContext &Ctx = *DAG.getContext();
3105  SDValue Cond = N->getOperand(0);
3106 
3107  if (N->getOpcode() != ISD::VSELECT)
3108  return SDValue();
3109 
3110  if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3111  return SDValue();
3112 
3113  // If this is a splitted VSELECT that was previously already handled, do
3114  // nothing.
3115  EVT CondVT = Cond->getValueType(0);
3116  if (CondVT.getScalarSizeInBits() != 1)
3117  return SDValue();
3118 
3119  EVT VSelVT = N->getValueType(0);
3120  // Only handle vector types which are a power of 2.
3121  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3122  return SDValue();
3123 
3124  // Don't touch if this will be scalarized.
3125  EVT FinalVT = VSelVT;
3126  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3127  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3128 
3129  if (FinalVT.getVectorNumElements() == 1)
3130  return SDValue();
3131 
3132  // If there is support for an i1 vector mask, don't touch.
3133  if (Cond.getOpcode() == ISD::SETCC) {
3134  EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3135  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3136  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3137  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3138  if (SetCCResVT.getScalarSizeInBits() == 1)
3139  return SDValue();
3140  } else if (CondVT.getScalarType() == MVT::i1) {
3141  // If there is support for an i1 vector mask (or only scalar i1 conditions),
3142  // don't touch.
3143  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3144  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3145 
3146  if (CondVT.getScalarType() == MVT::i1)
3147  return SDValue();
3148  }
3149 
3150  // Get the VT and operands for VSELECT, and widen if needed.
3151  SDValue VSelOp1 = N->getOperand(1);
3152  SDValue VSelOp2 = N->getOperand(2);
3153  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3154  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3155  VSelOp1 = GetWidenedVector(VSelOp1);
3156  VSelOp2 = GetWidenedVector(VSelOp2);
3157  }
3158 
3159  // The mask of the VSELECT should have integer elements.
3160  EVT ToMaskVT = VSelVT;
3161  if (!ToMaskVT.getScalarType().isInteger())
3162  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3163 
3164  SDValue Mask;
3165  if (Cond->getOpcode() == ISD::SETCC) {
3166  EVT MaskVT = getSETCCWidenedResultTy(Cond);
3167  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3168  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3169  Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3170  Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3171  // Cond is (AND/OR/XOR (SETCC, SETCC))
3172  SDValue SETCC0 = Cond->getOperand(0);
3173  SDValue SETCC1 = Cond->getOperand(1);
3174  EVT VT0 = getSETCCWidenedResultTy(SETCC0);
3175  EVT VT1 = getSETCCWidenedResultTy(SETCC1);
3176  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3177  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3178  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3179  EVT MaskVT;
3180  // If the two SETCCs have different VTs, either extend/truncate one of
3181  // them to the other "towards" ToMaskVT, or truncate one and extend the
3182  // other to ToMaskVT.
3183  if (ScalarBits0 != ScalarBits1) {
3184  EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3185  EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3186  if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3187  MaskVT = WideVT;
3188  else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3189  MaskVT = NarrowVT;
3190  else
3191  MaskVT = ToMaskVT;
3192  } else
3193  // If the two SETCCs have the same VT, don't change it.
3194  MaskVT = VT0;
3195 
3196  // Make new SETCCs and logical nodes.
3197  SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3198  SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3199  Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3200 
3201  // Convert the logical op for VSELECT if needed.
3202  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3203  } else
3204  return SDValue();
3205 
3206  return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3207 }
3208 
3209 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3210  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3211  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3212 
3213  SDValue Cond1 = N->getOperand(0);
3214  EVT CondVT = Cond1.getValueType();
3215  if (CondVT.isVector()) {
3216  if (SDValue Res = WidenVSELECTAndMask(N))
3217  return Res;
3218 
3219  EVT CondEltVT = CondVT.getVectorElementType();
3220  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3221  CondEltVT, WidenNumElts);
3222  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3223  Cond1 = GetWidenedVector(Cond1);
3224 
3225  // If we have to split the condition there is no point in widening the
3226  // select. This would result in an cycle of widening the select ->
3227  // widening the condition operand -> splitting the condition operand ->
3228  // splitting the select -> widening the select. Instead split this select
3229  // further and widen the resulting type.
3230  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3231  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3232  SDValue Res = ModifyToType(SplitSelect, WidenVT);
3233  return Res;
3234  }
3235 
3236  if (Cond1.getValueType() != CondWidenVT)
3237  Cond1 = ModifyToType(Cond1, CondWidenVT);
3238  }
3239 
3240  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3241  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3242  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3243  return DAG.getNode(N->getOpcode(), SDLoc(N),
3244  WidenVT, Cond1, InOp1, InOp2);
3245 }
3246 
3247 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3248  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3249  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3250  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3251  InOp1.getValueType(), N->getOperand(0),
3252  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3253 }
3254 
3255 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3256  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3257  return DAG.getUNDEF(WidenVT);
3258 }
3259 
3260 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3261  EVT VT = N->getValueType(0);
3262  SDLoc dl(N);
3263 
3264  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3265  unsigned NumElts = VT.getVectorNumElements();
3266  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3267 
3268  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3269  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3270 
3271  // Adjust mask based on new input vector length.
3272  SmallVector<int, 16> NewMask;
3273  for (unsigned i = 0; i != NumElts; ++i) {
3274  int Idx = N->getMaskElt(i);
3275  if (Idx < (int)NumElts)
3276  NewMask.push_back(Idx);
3277  else
3278  NewMask.push_back(Idx - NumElts + WidenNumElts);
3279  }
3280  for (unsigned i = NumElts; i != WidenNumElts; ++i)
3281  NewMask.push_back(-1);
3282  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
3283 }
3284 
3285 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
3286  assert(N->getValueType(0).isVector() &&
3287  N->getOperand(0).getValueType().isVector() &&
3288  "Operands must be vectors");
3289  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3290  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3291 
3292  SDValue InOp1 = N->getOperand(0);
3293  EVT InVT = InOp1.getValueType();
3294  assert(InVT.isVector() && "can not widen non-vector type");
3295  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
3296  InVT.getVectorElementType(), WidenNumElts);
3297 
3298  // The input and output types often differ here, and it could be that while
3299  // we'd prefer to widen the result type, the input operands have been split.
3300  // In this case, we also need to split the result of this node as well.
3301  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
3302  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
3303  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
3304  return Res;
3305  }
3306 
3307  InOp1 = GetWidenedVector(InOp1);
3308  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3309 
3310  // Assume that the input and output will be widen appropriately. If not,
3311  // we will have to unroll it at some point.
3312  assert(InOp1.getValueType() == WidenInVT &&
3313  InOp2.getValueType() == WidenInVT &&
3314  "Input not widened to expected type!");
3315  (void)WidenInVT;
3316  return DAG.getNode(ISD::SETCC, SDLoc(N),
3317  WidenVT, InOp1, InOp2, N->getOperand(2));
3318 }
3319 
3320 
3321 //===----------------------------------------------------------------------===//
3322 // Widen Vector Operand
3323 //===----------------------------------------------------------------------===//
3324 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
3325  LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
3326  dbgs() << "\n");
3327  SDValue Res = SDValue();
3328 
3329  // See if the target wants to custom widen this node.
3330  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3331  return false;
3332 
3333  switch (N->getOpcode()) {
3334  default:
3335 #ifndef NDEBUG
3336  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
3337  N->dump(&DAG);
3338  dbgs() << "\n";
3339 #endif
3340  llvm_unreachable("Do not know how to widen this operator's operand!");
3341 
3342  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
3343  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
3344  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
3345  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
3346  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
3347  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
3348  case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
3349  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
3350  case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
3351 
3352  case ISD::ANY_EXTEND:
3353  case ISD::SIGN_EXTEND:
3354  case ISD::ZERO_EXTEND:
3355  Res = WidenVecOp_EXTEND(N);
3356  break;
3357 
3358  case ISD::FP_EXTEND:
3359  case ISD::FP_TO_SINT:
3360  case ISD::FP_TO_UINT:
3361  case ISD::SINT_TO_FP:
3362  case ISD::UINT_TO_FP:
3363  case ISD::TRUNCATE:
3364  Res = WidenVecOp_Convert(N);
3365  break;
3366  }
3367 
3368  // If Res is null, the sub-method took care of registering the result.
3369  if (!Res.getNode()) return false;
3370 
3371  // If the result is N, the sub-method updated N in place. Tell the legalizer
3372  // core about this.
3373  if (Res.getNode() == N)
3374  return true;
3375 
3376 
3377  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3378  "Invalid operand expansion");
3379 
3380  ReplaceValueWith(SDValue(N, 0), Res);
3381  return false;
3382 }
3383 
3384 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
3385  SDLoc DL(N);
3386  EVT VT = N->getValueType(0);
3387 
3388  SDValue InOp = N->getOperand(0);
3389  assert(getTypeAction(InOp.getValueType()) ==
3391  "Unexpected type action");
3392  InOp = GetWidenedVector(InOp);
3395  "Input wasn't widened!");
3396 
3397  // We may need to further widen the operand until it has the same total
3398  // vector size as the result.
3399  EVT InVT = InOp.getValueType();
3400  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
3401  EVT InEltVT = InVT.getVectorElementType();
3402  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
3403  EVT FixedVT = (MVT::SimpleValueType)i;
3404  EVT FixedEltVT = FixedVT.getVectorElementType();
3405  if (TLI.isTypeLegal(FixedVT) &&
3406  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
3407  FixedEltVT == InEltVT) {
3408  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
3409  "Not enough elements in the fixed type for the operand!");
3410  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
3411  "We can't have the same type as we started with!");
3412  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
3413  InOp = DAG.getNode(
3414  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
3415  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3416  else
3417  InOp = DAG.getNode(
3418  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
3419  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3420  break;
3421  }
3422  }
3423  InVT = InOp.getValueType();
3424  if (InVT.getSizeInBits() != VT.getSizeInBits())
3425  // We couldn't find a legal vector type that was a widening of the input
3426  // and could be extended in-register to the result type, so we have to
3427  // scalarize.
3428  return WidenVecOp_Convert(N);
3429  }
3430 
3431  // Use special DAG nodes to represent the operation of extending the
3432  // low lanes.
3433  switch (N->getOpcode()) {
3434  default:
3435  llvm_unreachable("Extend legalization on extend operation!");
3436  case ISD::ANY_EXTEND:
3437  return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
3438  case ISD::SIGN_EXTEND:
3439  return DAG.getSignExtendVectorInReg(InOp, DL, VT);
3440  case ISD::ZERO_EXTEND:
3441  return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
3442  }
3443 }
3444 
3445 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
3446  // The result (and first input) is legal, but the second input is illegal.
3447  // We can't do much to fix that, so just unroll and let the extracts off of
3448  // the second input be widened as needed later.
3449  return DAG.UnrollVectorOp(N);
3450 }
3451 
3452 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
3453  // Since the result is legal and the input is illegal.
3454  EVT VT = N->getValueType(0);
3455  EVT EltVT = VT.getVectorElementType();
3456  SDLoc dl(N);
3457  unsigned NumElts = VT.getVectorNumElements();
3458  SDValue InOp = N->getOperand(0);
3459  assert(getTypeAction(InOp.getValueType()) ==
3461  "Unexpected type action");
3462  InOp = GetWidenedVector(InOp);
3463  EVT InVT = InOp.getValueType();
3464  unsigned Opcode = N->getOpcode();
3465 
3466  // See if a widened result type would be legal, if so widen the node.
3467  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3468  InVT.getVectorNumElements());
3469  if (TLI.isTypeLegal(WideVT)) {
3470  SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp);
3471  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
3472  DAG.getIntPtrConstant(0, dl));
3473  }
3474 
3475  EVT InEltVT = InVT.getVectorElementType();
3476 
3477  // Unroll the convert into some scalar code and create a nasty build vector.
3478  SmallVector<SDValue, 16> Ops(NumElts);
3479  for (unsigned i=0; i < NumElts; ++i)
3480  Ops[i] = DAG.getNode(
3481  Opcode, dl, EltVT,
3482  DAG.getNode(
3483  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
3484  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3485 
3486  return DAG.getBuildVector(VT, dl, Ops);
3487 }
3488 
3489 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
3490  EVT VT = N->getValueType(0);
3491  SDValue InOp = GetWidenedVector(N->getOperand(0));
3492  EVT InWidenVT = InOp.getValueType();
3493  SDLoc dl(N);
3494 
3495  // Check if we can convert between two legal vector types and extract.
3496  unsigned InWidenSize = InWidenVT.getSizeInBits();
3497  unsigned Size = VT.getSizeInBits();
3498  // x86mmx is not an acceptable vector element type, so don't try.
3499  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
3500  unsigned NewNumElts = InWidenSize / Size;
3501  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
3502  if (TLI.isTypeLegal(NewVT)) {
3503  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
3504  return DAG.getNode(
3505  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
3506  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3507  }
3508  }
3509 
3510  return CreateStackStoreLoad(InOp, VT);
3511 }
3512 
3513 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
3514  // If the input vector is not legal, it is likely that we will not find a
3515  // legal vector of the same size. Replace the concatenate vector with a
3516  // nasty build vector.
3517  EVT VT = N->getValueType(0);
3518  EVT EltVT = VT.getVectorElementType();
3519  SDLoc dl(N);
3520  unsigned NumElts = VT.getVectorNumElements();
3521  SmallVector<SDValue, 16> Ops(NumElts);
3522 
3523  EVT InVT = N->getOperand(0).getValueType();
3524  unsigned NumInElts = InVT.getVectorNumElements();
3525 
3526  unsigned Idx = 0;
3527  unsigned NumOperands = N->getNumOperands();
3528  for (unsigned i=0; i < NumOperands; ++i) {
3529  SDValue InOp = N->getOperand(i);
3530  assert(getTypeAction(InOp.getValueType()) ==
3532  "Unexpected type action");
3533  InOp = GetWidenedVector(InOp);
3534  for (unsigned j=0; j < NumInElts; ++j)
3535  Ops[Idx++] = DAG.getNode(
3536  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3537  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3538  }
3539  return DAG.getBuildVector(VT, dl, Ops);
3540 }
3541 
3542 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3543  SDValue InOp = GetWidenedVector(N->getOperand(0));
3544  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
3545  N->getValueType(0), InOp, N->getOperand(1));
3546 }
3547 
3548 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3549  SDValue InOp = GetWidenedVector(N->getOperand(0));
3550  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3551  N->getValueType(0), InOp, N->getOperand(1));
3552 }
3553 
3554 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
3555  // We have to widen the value, but we want only to store the original
3556  // vector type.
3557  StoreSDNode *ST = cast<StoreSDNode>(N);
3558 
3559  if (!ST->getMemoryVT().getScalarType().isByteSized())
3560  return TLI.scalarizeVectorStore(ST, DAG);
3561 
3562  SmallVector<SDValue, 16> StChain;
3563  if (ST->isTruncatingStore())
3564  GenWidenVectorTruncStores(StChain, ST);
3565  else
3566  GenWidenVectorStores(StChain, ST);
3567 
3568  if (StChain.size() == 1)
3569  return StChain[0];
3570  else
3571  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
3572 }
3573 
3574 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
3575  assert(OpNo == 3 && "Can widen only data operand of mstore");
3576  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
3577  SDValue Mask = MST->getMask();
3578  EVT MaskVT = Mask.getValueType();
3579  SDValue StVal = MST->getValue();
3580  // Widen the value
3581  SDValue WideVal = GetWidenedVector(StVal);
3582  SDLoc dl(N);
3583 
3584  // The mask should be widened as well.
3585  EVT WideVT = WideVal.getValueType();
3586  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3587  MaskVT.getVectorElementType(),
3588  WideVT.getVectorNumElements());
3589  Mask = ModifyToType(Mask, WideMaskVT, true);
3590 
3591  assert(Mask.getValueType().getVectorNumElements() ==
3592  WideVal.getValueType().getVectorNumElements() &&
3593  "Mask and data vectors should have the same number of elements");
3594  return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
3595  Mask, MST->getMemoryVT(), MST->getMemOperand(),
3596  false, MST->isCompressingStore());
3597 }
3598 
3599 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
3600  assert(OpNo == 1 && "Can widen only data operand of mscatter");
3601  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
3602  SDValue DataOp = MSC->getValue();
3603  SDValue Mask = MSC->getMask();
3604  EVT MaskVT = Mask.getValueType();
3605  SDValue Scale = MSC->getScale();
3606 
3607  // Widen the value.
3608  SDValue WideVal = GetWidenedVector(DataOp);
3609  EVT WideVT = WideVal.getValueType();
3610  unsigned NumElts = WideVT.getVectorNumElements();
3611  SDLoc dl(N);
3612 
3613  // The mask should be widened as well.
3614  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3615  MaskVT.getVectorElementType(), NumElts);
3616  Mask = ModifyToType(Mask, WideMaskVT, true);
3617 
3618  // Widen index.
3619  SDValue Index = MSC->getIndex();
3620  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3621  Index.getValueType().getScalarType(),
3622  NumElts);
3623  Index = ModifyToType(Index, WideIndexVT);
3624 
3625  SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index,
3626  Scale};
3627  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
3628  MSC->getMemoryVT(), dl, Ops,
3629  MSC->getMemOperand());
3630 }
3631 
3632 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
3633  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
3634  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3635  SDLoc dl(N);
3636  EVT VT = N->getValueType(0);
3637 
3638  // WARNING: In this code we widen the compare instruction with garbage.
3639  // This garbage may contain denormal floats which may be slow. Is this a real
3640  // concern ? Should we zero the unused lanes if this is a float compare ?
3641 
3642  // Get a new SETCC node to compare the newly widened operands.
3643  // Only some of the compared elements are legal.
3644  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3645  InOp0.getValueType());
3646  // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
3647  if (VT.getScalarType() == MVT::i1)
3648  SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
3649  SVT.getVectorNumElements());
3650 
3651  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
3652  SVT, InOp0, InOp1, N->getOperand(2));
3653 
3654  // Extract the needed results from the result vector.
3655  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
3656  SVT.getVectorElementType(),
3657  VT.getVectorNumElements());
3658  SDValue CC = DAG.getNode(
3659  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
3660  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3661 
3662  return PromoteTargetBoolean(CC, VT);
3663 }
3664 
3665 
3666 //===----------------------------------------------------------------------===//
3667 // Vector Widening Utilities
3668 //===----------------------------------------------------------------------===//
3669 
3670 // Utility function to find the type to chop up a widen vector for load/store
3671 // TLI: Target lowering used to determine legal types.
3672 // Width: Width left need to load/store.
3673 // WidenVT: The widen vector type to load to/store from
3674 // Align: If 0, don't allow use of a wider type
3675 // WidenEx: If Align is not 0, the amount additional we can load/store from.
3676 
3677 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
3678  unsigned Width, EVT WidenVT,
3679  unsigned Align = 0, unsigned WidenEx = 0) {
3680  EVT WidenEltVT = WidenVT.getVectorElementType();
3681  unsigned WidenWidth = WidenVT.getSizeInBits();
3682  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
3683  unsigned AlignInBits = Align*8;
3684 
3685  // If we have one element to load/store, return it.
3686  EVT RetVT = WidenEltVT;
3687  if (Width == WidenEltWidth)
3688  return RetVT;
3689 
3690  // See if there is larger legal integer than the element type to load/store.
3691  unsigned VT;
3692  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
3693  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
3694  EVT MemVT((MVT::SimpleValueType) VT);
3695  unsigned MemVTWidth = MemVT.getSizeInBits();
3696  if (MemVT.getSizeInBits() <= WidenEltWidth)
3697  break;
3698  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
3699  if ((Action == TargetLowering::TypeLegal ||
3701  (WidenWidth % MemVTWidth) == 0 &&
3702  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3703  (MemVTWidth <= Width ||
3704  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3705  RetVT = MemVT;
3706  break;
3707  }
3708  }
3709 
3710  // See if there is a larger vector type to load/store that has the same vector
3711  // element type and is evenly divisible with the WidenVT.
3712  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
3713  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
3714  EVT MemVT = (MVT::SimpleValueType) VT;
3715  unsigned MemVTWidth = MemVT.getSizeInBits();
3716  if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
3717  (WidenWidth % MemVTWidth) == 0 &&
3718  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3719  (MemVTWidth <= Width ||
3720  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3721  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
3722  return MemVT;
3723  }
3724  }
3725 
3726  return RetVT;
3727 }
3728 
3729 // Builds a vector type from scalar loads
3730 // VecTy: Resulting Vector type
3731 // LDOps: Load operators to build a vector type
3732 // [Start,End) the list of loads to use.
3734  SmallVectorImpl<SDValue> &LdOps,
3735  unsigned Start, unsigned End) {
3736  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3737  SDLoc dl(LdOps[Start]);
3738  EVT LdTy = LdOps[Start].getValueType();
3739  unsigned Width = VecTy.getSizeInBits();
3740  unsigned NumElts = Width / LdTy.getSizeInBits();
3741  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
3742 
3743  unsigned Idx = 1;
3744  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
3745 
3746  for (unsigned i = Start + 1; i != End; ++i) {
3747  EVT NewLdTy = LdOps[i].getValueType();
3748  if (NewLdTy != LdTy) {
3749  NumElts = Width / NewLdTy.getSizeInBits();
3750  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
3751  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
3752  // Readjust position and vector position based on new load type.
3753  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
3754  LdTy = NewLdTy;
3755  }
3756  VecOp = DAG.getNode(
3757  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
3758  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3759  }
3760  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
3761 }
3762 
3763 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
3764  LoadSDNode *LD) {
3765  // The strategy assumes that we can efficiently load power-of-two widths.
3766  // The routine chops the vector into the largest vector loads with the same
3767  // element type or scalar loads and then recombines it to the widen vector
3768  // type.
3769  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3770  unsigned WidenWidth = WidenVT.getSizeInBits();
3771  EVT LdVT = LD->getMemoryVT();
3772  SDLoc dl(LD);
3773  assert(LdVT.isVector() && WidenVT.isVector());
3774  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
3775 
3776  // Load information
3777  SDValue Chain = LD->getChain();
3778  SDValue BasePtr = LD->getBasePtr();
3779  unsigned Align = LD->getAlignment();
3780  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3781  AAMDNodes AAInfo = LD->getAAInfo();
3782 
3783  int LdWidth = LdVT.getSizeInBits();
3784  int WidthDiff = WidenWidth - LdWidth;
3785  unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
3786 
3787  // Find the vector type that can load from.
3788  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3789  int NewVTWidth = NewVT.getSizeInBits();
3790  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
3791  Align, MMOFlags, AAInfo);
3792  LdChain.push_back(LdOp.getValue(1));
3793 
3794  // Check if we can load the element with one instruction.
3795  if (LdWidth <= NewVTWidth) {
3796  if (!NewVT.isVector()) {
3797  unsigned NumElts = WidenWidth / NewVTWidth;
3798  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
3799  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
3800  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
3801  }
3802  if (NewVT == WidenVT)
3803  return LdOp;
3804 
3805  assert(WidenWidth % NewVTWidth == 0);
3806  unsigned NumConcat = WidenWidth / NewVTWidth;
3807  SmallVector<SDValue, 16> ConcatOps(NumConcat);
3808  SDValue UndefVal = DAG.getUNDEF(NewVT);
3809  ConcatOps[0] = LdOp;
3810  for (unsigned i = 1; i != NumConcat; ++i)
3811  ConcatOps[i] = UndefVal;
3812  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
3813  }
3814 
3815  // Load vector by using multiple loads from largest vector to scalar.
3817  LdOps.push_back(LdOp);
3818 
3819  LdWidth -= NewVTWidth;
3820  unsigned Offset = 0;
3821 
3822  while (LdWidth > 0) {
3823  unsigned Increment = NewVTWidth / 8;
3824  Offset += Increment;
3825  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
3826 
3827  SDValue L;
3828  if (LdWidth < NewVTWidth) {
3829  // The current type we are using is too large. Find a better size.
3830  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3831  NewVTWidth = NewVT.getSizeInBits();
3832  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3833  LD->getPointerInfo().getWithOffset(Offset),
3834  MinAlign(Align, Increment), MMOFlags, AAInfo);
3835  LdChain.push_back(L.getValue(1));
3836  if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
3837  // Later code assumes the vector loads produced will be mergeable, so we
3838  // must pad the final entry up to the previous width. Scalars are
3839  // combined separately.
3841  Loads.push_back(L);
3842  unsigned size = L->getValueSizeInBits(0);
3843  while (size < LdOp->getValueSizeInBits(0)) {
3844  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
3845  size += L->getValueSizeInBits(0);
3846  }
3847  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
3848  }
3849  } else {
3850  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3851  LD->getPointerInfo().getWithOffset(Offset),
3852  MinAlign(Align, Increment), MMOFlags, AAInfo);
3853  LdChain.push_back(L.getValue(1));
3854  }
3855 
3856  LdOps.push_back(L);
3857  LdOp = L;
3858 
3859  LdWidth -= NewVTWidth;
3860  }
3861 
3862  // Build the vector from the load operations.
3863  unsigned End = LdOps.size();
3864  if (!LdOps[0].getValueType().isVector())
3865  // All the loads are scalar loads.
3866  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
3867 
3868  // If the load contains vectors, build the vector using concat vector.
3869  // All of the vectors used to load are power-of-2, and the scalar loads can be
3870  // combined to make a power-of-2 vector.
3871  SmallVector<SDValue, 16> ConcatOps(End);
3872  int i = End - 1;
3873  int Idx = End;
3874  EVT LdTy = LdOps[i].getValueType();
3875  // First, combine the scalar loads to a vector.
3876  if (!LdTy.isVector()) {
3877  for (--i; i >= 0; --i) {
3878  LdTy = LdOps[i].getValueType();
3879  if (LdTy.isVector())
3880  break;
3881  }
3882  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
3883  }
3884  ConcatOps[--Idx] = LdOps[i];
3885  for (--i; i >= 0; --i) {
3886  EVT NewLdTy = LdOps[i].getValueType();
3887  if (NewLdTy != LdTy) {
3888  // Create a larger vector.
3889  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
3890  makeArrayRef(&ConcatOps[Idx], End - Idx));
3891  Idx = End - 1;
3892  LdTy = NewLdTy;
3893  }
3894  ConcatOps[--Idx] = LdOps[i];
3895  }
3896 
3897  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
3898  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3899  makeArrayRef(&ConcatOps[Idx], End - Idx));
3900 
3901  // We need to fill the rest with undefs to build the vector.
3902  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
3903  SmallVector<SDValue, 16> WidenOps(NumOps);
3904  SDValue UndefVal = DAG.getUNDEF(LdTy);
3905  {
3906  unsigned i = 0;
3907  for (; i != End-Idx; ++i)
3908  WidenOps[i] = ConcatOps[Idx+i];
3909  for (; i != NumOps; ++i)
3910  WidenOps[i] = UndefVal;
3911  }
3912  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
3913 }
3914 
3915 SDValue
3916 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
3917  LoadSDNode *LD,
3918  ISD::LoadExtType ExtType) {
3919  // For extension loads, it may not be more efficient to chop up the vector
3920  // and then extend it. Instead, we unroll the load and build a new vector.
3921  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3922  EVT LdVT = LD->getMemoryVT();
3923  SDLoc dl(LD);
3924  assert(LdVT.isVector() && WidenVT.isVector());
3925 
3926  // Load information
3927  SDValue Chain = LD->getChain();
3928  SDValue BasePtr = LD->getBasePtr();
3929  unsigned Align = LD->getAlignment();
3930  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
3931  AAMDNodes AAInfo = LD->getAAInfo();
3932 
3933  EVT EltVT = WidenVT.getVectorElementType();
3934  EVT LdEltVT = LdVT.getVectorElementType();
3935  unsigned NumElts = LdVT.getVectorNumElements();
3936 
3937  // Load each element and widen.
3938  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3939  SmallVector<SDValue, 16> Ops(WidenNumElts);
3940  unsigned Increment = LdEltVT.getSizeInBits() / 8;
3941  Ops[0] =
3942  DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
3943  LdEltVT, Align, MMOFlags, AAInfo);
3944  LdChain.push_back(Ops[0].getValue(1));
3945  unsigned i = 0, Offset = Increment;
3946  for (i=1; i < NumElts; ++i, Offset += Increment) {
3947  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
3948  Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
3949  LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
3950  Align, MMOFlags, AAInfo);
3951  LdChain.push_back(Ops[i].getValue(1));
3952  }
3953 
3954  // Fill the rest with undefs.
3955  SDValue UndefVal = DAG.getUNDEF(EltVT);
3956  for (; i != WidenNumElts; ++i)
3957  Ops[i] = UndefVal;
3958 
3959  return DAG.getBuildVector(WidenVT, dl, Ops);
3960 }
3961 
3962 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
3963  StoreSDNode *ST) {
3964  // The strategy assumes that we can efficiently store power-of-two widths.
3965  // The routine chops the vector into the largest vector stores with the same
3966  // element type or scalar stores.
3967  SDValue Chain = ST->getChain();
3968  SDValue BasePtr = ST->getBasePtr();
3969  unsigned Align = ST->getAlignment();
3970  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
3971  AAMDNodes AAInfo = ST->getAAInfo();
3972  SDValue ValOp = GetWidenedVector(ST->getValue());
3973  SDLoc dl(ST);
3974 
3975  EVT StVT = ST->getMemoryVT();
3976  unsigned StWidth = StVT.getSizeInBits();
3977  EVT ValVT = ValOp.getValueType();
3978  unsigned ValWidth = ValVT.getSizeInBits();
3979  EVT ValEltVT = ValVT.getVectorElementType();
3980  unsigned ValEltWidth = ValEltVT.getSizeInBits();
3981  assert(StVT.getVectorElementType() == ValEltVT);
3982 
3983  int Idx = 0; // current index to store
3984  unsigned Offset = 0; // offset from base to store
3985  while (StWidth != 0) {
3986  // Find the largest vector type we can store with.
3987  EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
3988  unsigned NewVTWidth = NewVT.getSizeInBits();
3989  unsigned Increment = NewVTWidth / 8;
3990  if (NewVT.isVector()) {
3991  unsigned NumVTElts = NewVT.getVectorNumElements();
3992  do {
3993  SDValue EOp = DAG.getNode(
3994  ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
3995  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3996  StChain.push_back(DAG.getStore(
3997  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
3998  MinAlign(Align, Offset), MMOFlags, AAInfo));
3999  StWidth -= NewVTWidth;
4000  Offset += Increment;
4001  Idx += NumVTElts;
4002 
4003  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4004  } while (StWidth != 0 && StWidth >= NewVTWidth);
4005  } else {
4006  // Cast the vector to the scalar type we can store.
4007  unsigned NumElts = ValWidth / NewVTWidth;
4008  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4009  SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4010  // Readjust index position based on new vector type.
4011  Idx = Idx * ValEltWidth / NewVTWidth;
4012  do {
4013  SDValue EOp = DAG.getNode(
4014  ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4015  DAG.getConstant(Idx++, dl,
4016  TLI.getVectorIdxTy(DAG.getDataLayout())));
4017  StChain.push_back(DAG.getStore(
4018  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4019  MinAlign(Align, Offset), MMOFlags, AAInfo));
4020  StWidth -= NewVTWidth;
4021  Offset += Increment;
4022  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4023  } while (StWidth != 0 && StWidth >= NewVTWidth);
4024  // Restore index back to be relative to the original widen element type.
4025  Idx = Idx * NewVTWidth / ValEltWidth;
4026  }
4027  }
4028 }
4029 
4030 void
4031 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4032  StoreSDNode *ST) {
4033  // For extension loads, it may not be more efficient to truncate the vector
4034  // and then store it. Instead, we extract each element and then store it.
4035  SDValue Chain = ST->getChain();
4036  SDValue BasePtr = ST->getBasePtr();
4037  unsigned Align = ST->getAlignment();
4038  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4039  AAMDNodes AAInfo = ST->getAAInfo();
4040  SDValue ValOp = GetWidenedVector(ST->getValue());
4041  SDLoc dl(ST);
4042 
4043  EVT StVT = ST->getMemoryVT();
4044  EVT ValVT = ValOp.getValueType();
4045 
4046  // It must be true that the wide vector type is bigger than where we need to
4047  // store.
4048  assert(StVT.isVector() && ValOp.getValueType().isVector());
4049  assert(StVT.bitsLT(ValOp.getValueType()));
4050 
4051  // For truncating stores, we can not play the tricks of chopping legal vector
4052  // types and bitcast it to the right type. Instead, we unroll the store.
4053  EVT StEltVT = StVT.getVectorElementType();
4054  EVT ValEltVT = ValVT.getVectorElementType();
4055  unsigned Increment = ValEltVT.getSizeInBits() / 8;
4056  unsigned NumElts = StVT.getVectorNumElements();
4057  SDValue EOp = DAG.getNode(
4058  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4059  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4060  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4061  ST->getPointerInfo(), StEltVT, Align,
4062  MMOFlags, AAInfo));
4063  unsigned Offset = Increment;
4064  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4065  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4066  SDValue EOp = DAG.getNode(
4067  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4068  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4069  StChain.push_back(DAG.getTruncStore(
4070  Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4071  StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4072  }
4073 }
4074 
4075 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
4076 /// input vector must have the same element type as NVT.
4077 /// FillWithZeroes specifies that the vector should be widened with zeroes.
4078 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
4079  bool FillWithZeroes) {
4080  // Note that InOp might have been widened so it might already have
4081  // the right width or it might need be narrowed.
4082  EVT InVT = InOp.getValueType();
4084  "input and widen element type must match");
4085  SDLoc dl(InOp);
4086 
4087  // Check if InOp already has the right width.
4088  if (InVT == NVT)
4089  return InOp;
4090 
4091  unsigned InNumElts = InVT.getVectorNumElements();
4092  unsigned WidenNumElts = NVT.getVectorNumElements();
4093  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
4094  unsigned NumConcat = WidenNumElts / InNumElts;
4095  SmallVector<SDValue, 16> Ops(NumConcat);
4096  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
4097  DAG.getUNDEF(InVT);
4098  Ops[0] = InOp;
4099  for (unsigned i = 1; i != NumConcat; ++i)
4100  Ops[i] = FillVal;
4101 
4102  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
4103  }
4104 
4105  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
4106  return DAG.getNode(
4107  ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
4108  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4109 
4110  // Fall back to extract and build.
4111  SmallVector<SDValue, 16> Ops(WidenNumElts);
4112  EVT EltVT = NVT.getVectorElementType();
4113  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
4114  unsigned Idx;
4115  for (Idx = 0; Idx < MinNumElts; ++Idx)
4116  Ops[Idx] = DAG.getNode(
4117  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4118  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4119 
4120  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
4121  DAG.getUNDEF(EltVT);
4122  for ( ; Idx < WidenNumElts; ++Idx)
4123  Ops[Idx] = FillVal;
4124  return DAG.getBuildVector(NVT, dl, Ops);
4125 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:539
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:506
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:563
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isUndef() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:334
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:115
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:343
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:137
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:487
SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:351
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getBasePtr() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getChain() const
unsigned getAlignment() const
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:964
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:478
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:360
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:329
uint64_t High
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:405
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
const SDValue & getValue() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
A description of a memory reference used in the backend.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:380
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
This is used by foldAnyOrAllBitsSet() to capture a source value (Root) and the bit indexes (Mask) nee...
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:457
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:398
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:448
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
const SDValue & getScale() const
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:399
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:865
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:571
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:465
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:492
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:303
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:229
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:610
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getBasePtr() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
These reductions are non-strict, and have a single vector operand.
Definition: ISDOpcodes.h:820
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:429
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:69
Simple binary floating point operators.
Definition: ISDOpcodes.h:260
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:886
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:308
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node&#39;s operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:732
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:434
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition: ValueTypes.h:58
bool hasNoNaNs() const
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:386
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:524
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
const SDValue & getSrc0() const
bool isVolatile() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const SDValue & getMask() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
The memory access writes data.
SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:401
auto size(R &&Range, typename std::enable_if< std::is_same< typename std::iterator_traits< decltype(Range.begin())>::iterator_category, std::random_access_iterator_tag >::value, void >::type *=nullptr) -> decltype(std::distance(Range.begin(), Range.end()))
Get the size of a range.
Definition: STLExtras.h:1032
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:295
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:315
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:222
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:521
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:729
bool isVector(MCInstrInfo const &MCII, MCInst const &MCI)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:787
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:96
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:642
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:383
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
Definition: ValueTypes.h:342
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored...
Represents one node in the SelectionDAG.
static bool isSETCCorConvertedSETCC(SDValue N)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
MachinePointerInfo getWithOffset(int64_t O) const
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:924
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:390
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:438
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:441
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:288
int getMaskElt(unsigned Idx) const
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
Definition: ISDOpcodes.h:566
Flags
Flags values. These may be or&#39;d together.
amdgpu Simplify well known AMD library false Value Value * Arg
The memory access reads data.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
pointer data()
Return a pointer to the vector&#39;s buffer, even if empty().
Definition: SmallVector.h:144
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:456
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:575
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return &#39;Legal&#39;) or we ...
#define N
Flags getFlags() const
Return the raw flags of the source value,.
unsigned getOpcode() const
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:321
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MLOAD node.
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:278
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:825
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
bool isUndef() const
Return true if the type of the node type undefined.
ISD::LoadExtType getExtensionType() const
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:413
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
static bool isLogicalMaskOp(unsigned Opcode)
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:198
Conversion operators.
Definition: ISDOpcodes.h:435
BooleanContent
Enum that describes how the target represents true/false values.
const SDValue & getOperand(unsigned i) const
bool isBigEndian() const
Definition: DataLayout.h:222
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:444
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Definition: SelectionDAG.h:819
#define LLVM_DEBUG(X)
Definition: Debug.h:119
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:555
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVMContext * getContext() const
Definition: SelectionDAG.h:404
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:351
const SDValue & getMask() const
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:356
This class is used to represent ISD::LOAD nodes.