LLVM  3.7.0
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16 // types.
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "legalize-types"
30 
31 //===----------------------------------------------------------------------===//
32 // Result Vector Scalarization: <1 x ty> -> ty.
33 //===----------------------------------------------------------------------===//
34 
35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36  DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
37  N->dump(&DAG);
38  dbgs() << "\n");
39  SDValue R = SDValue();
40 
41  switch (N->getOpcode()) {
42  default:
43 #ifndef NDEBUG
44  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45  N->dump(&DAG);
46  dbgs() << "\n";
47 #endif
48  report_fatal_error("Do not know how to scalarize the result of this "
49  "operator!\n");
50 
51  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54  case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break;
55  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
56  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
57  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
58  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
59  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
60  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
61  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
62  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
63  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
64  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
65  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
66  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
67  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
68  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
69  case ISD::ANY_EXTEND:
70  case ISD::BSWAP:
71  case ISD::CTLZ:
73  case ISD::CTPOP:
74  case ISD::CTTZ:
76  case ISD::FABS:
77  case ISD::FCEIL:
78  case ISD::FCOS:
79  case ISD::FEXP:
80  case ISD::FEXP2:
81  case ISD::FFLOOR:
82  case ISD::FLOG:
83  case ISD::FLOG10:
84  case ISD::FLOG2:
85  case ISD::FNEARBYINT:
86  case ISD::FNEG:
87  case ISD::FP_EXTEND:
88  case ISD::FP_TO_SINT:
89  case ISD::FP_TO_UINT:
90  case ISD::FRINT:
91  case ISD::FROUND:
92  case ISD::FSIN:
93  case ISD::FSQRT:
94  case ISD::FTRUNC:
95  case ISD::SIGN_EXTEND:
96  case ISD::SINT_TO_FP:
97  case ISD::TRUNCATE:
98  case ISD::UINT_TO_FP:
99  case ISD::ZERO_EXTEND:
100  R = ScalarizeVecRes_UnaryOp(N);
101  break;
102 
103  case ISD::ADD:
104  case ISD::AND:
105  case ISD::FADD:
106  case ISD::FCOPYSIGN:
107  case ISD::FDIV:
108  case ISD::FMUL:
109  case ISD::FMINNUM:
110  case ISD::FMAXNUM:
111 
112  case ISD::FPOW:
113  case ISD::FREM:
114  case ISD::FSUB:
115  case ISD::MUL:
116  case ISD::OR:
117  case ISD::SDIV:
118  case ISD::SREM:
119  case ISD::SUB:
120  case ISD::UDIV:
121  case ISD::UREM:
122  case ISD::XOR:
123  case ISD::SHL:
124  case ISD::SRA:
125  case ISD::SRL:
126  R = ScalarizeVecRes_BinOp(N);
127  break;
128  case ISD::FMA:
129  R = ScalarizeVecRes_TernaryOp(N);
130  break;
131  }
132 
133  // If R is null, the sub-method took care of registering the result.
134  if (R.getNode())
135  SetScalarizedVector(SDValue(N, ResNo), R);
136 }
137 
138 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
139  SDValue LHS = GetScalarizedVector(N->getOperand(0));
140  SDValue RHS = GetScalarizedVector(N->getOperand(1));
141  return DAG.getNode(N->getOpcode(), SDLoc(N),
142  LHS.getValueType(), LHS, RHS);
143 }
144 
145 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
146  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
147  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
148  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
149  return DAG.getNode(N->getOpcode(), SDLoc(N),
150  Op0.getValueType(), Op0, Op1, Op2);
151 }
152 
153 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
154  unsigned ResNo) {
155  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
156  return GetScalarizedVector(Op);
157 }
158 
159 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
160  EVT NewVT = N->getValueType(0).getVectorElementType();
161  return DAG.getNode(ISD::BITCAST, SDLoc(N),
162  NewVT, N->getOperand(0));
163 }
164 
165 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
166  EVT EltVT = N->getValueType(0).getVectorElementType();
167  SDValue InOp = N->getOperand(0);
168  // The BUILD_VECTOR operands may be of wider element types and
169  // we may need to truncate them back to the requested return type.
170  if (EltVT.isInteger())
171  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
172  return InOp;
173 }
174 
175 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) {
176  EVT NewVT = N->getValueType(0).getVectorElementType();
177  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
178  return DAG.getConvertRndSat(NewVT, SDLoc(N),
179  Op0, DAG.getValueType(NewVT),
180  DAG.getValueType(Op0.getValueType()),
181  N->getOperand(3),
182  N->getOperand(4),
183  cast<CvtRndSatSDNode>(N)->getCvtCode());
184 }
185 
186 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
187  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
189  N->getOperand(0), N->getOperand(1));
190 }
191 
192 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
193  EVT NewVT = N->getValueType(0).getVectorElementType();
194  SDValue Op = GetScalarizedVector(N->getOperand(0));
195  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
196  NewVT, Op, N->getOperand(1));
197 }
198 
199 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
200  SDValue Op = GetScalarizedVector(N->getOperand(0));
201  return DAG.getNode(ISD::FPOWI, SDLoc(N),
202  Op.getValueType(), Op, N->getOperand(1));
203 }
204 
205 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
206  // The value to insert may have a wider type than the vector element type,
207  // so be sure to truncate it to the element type if necessary.
208  SDValue Op = N->getOperand(1);
209  EVT EltVT = N->getValueType(0).getVectorElementType();
210  if (Op.getValueType() != EltVT)
211  // FIXME: Can this happen for floating point types?
212  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
213  return Op;
214 }
215 
216 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
217  assert(N->isUnindexed() && "Indexed vector load?");
218 
219  SDValue Result = DAG.getLoad(ISD::UNINDEXED,
220  N->getExtensionType(),
222  SDLoc(N),
223  N->getChain(), N->getBasePtr(),
224  DAG.getUNDEF(N->getBasePtr().getValueType()),
225  N->getPointerInfo(),
227  N->isVolatile(), N->isNonTemporal(),
229  N->getAAInfo());
230 
231  // Legalized the chain result - switch anything that used the old chain to
232  // use the new one.
233  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
234  return Result;
235 }
236 
237 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
238  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
239  EVT DestVT = N->getValueType(0).getVectorElementType();
240  SDValue Op = N->getOperand(0);
241  EVT OpVT = Op.getValueType();
242  SDLoc DL(N);
243  // The result needs scalarizing, but it's not a given that the source does.
244  // This is a workaround for targets where it's impossible to scalarize the
245  // result of a conversion, because the source type is legal.
246  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
247  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
248  // legal and was not scalarized.
249  // See the similar logic in ScalarizeVecRes_VSETCC
250  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
251  Op = GetScalarizedVector(Op);
252  } else {
253  EVT VT = OpVT.getVectorElementType();
254  Op = DAG.getNode(
255  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
256  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
257  }
258  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
259 }
260 
261 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
262  EVT EltVT = N->getValueType(0).getVectorElementType();
263  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
264  SDValue LHS = GetScalarizedVector(N->getOperand(0));
265  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
266  LHS, DAG.getValueType(ExtVT));
267 }
268 
269 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
270  // If the operand is wider than the vector element type then it is implicitly
271  // truncated. Make that explicit here.
272  EVT EltVT = N->getValueType(0).getVectorElementType();
273  SDValue InOp = N->getOperand(0);
274  if (InOp.getValueType() != EltVT)
275  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
276  return InOp;
277 }
278 
279 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
280  SDValue Cond = GetScalarizedVector(N->getOperand(0));
281  SDValue LHS = GetScalarizedVector(N->getOperand(1));
282  TargetLowering::BooleanContent ScalarBool =
283  TLI.getBooleanContents(false, false);
284  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
285 
286  // If integer and float booleans have different contents then we can't
287  // reliably optimize in all cases. There is a full explanation for this in
288  // DAGCombiner::visitSELECT() where the same issue affects folding
289  // (select C, 0, 1) to (xor C, 1).
290  if (TLI.getBooleanContents(false, false) !=
291  TLI.getBooleanContents(false, true)) {
292  // At least try the common case where the boolean is generated by a
293  // comparison.
294  if (Cond->getOpcode() == ISD::SETCC) {
295  EVT OpVT = Cond->getOperand(0)->getValueType(0);
296  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
297  VecBool = TLI.getBooleanContents(OpVT);
298  } else
300  }
301 
302  if (ScalarBool != VecBool) {
303  EVT CondVT = Cond.getValueType();
304  switch (ScalarBool) {
306  break;
308  assert(VecBool == TargetLowering::UndefinedBooleanContent ||
310  // Vector read from all ones, scalar expects a single 1 so mask.
311  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
312  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
313  break;
315  assert(VecBool == TargetLowering::UndefinedBooleanContent ||
317  // Vector reads from a one, scalar from all ones so sign extend.
318  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
319  Cond, DAG.getValueType(MVT::i1));
320  break;
321  }
322  }
323 
324  return DAG.getSelect(SDLoc(N),
325  LHS.getValueType(), Cond, LHS,
326  GetScalarizedVector(N->getOperand(2)));
327 }
328 
329 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
330  SDValue LHS = GetScalarizedVector(N->getOperand(1));
331  return DAG.getSelect(SDLoc(N),
332  LHS.getValueType(), N->getOperand(0), LHS,
333  GetScalarizedVector(N->getOperand(2)));
334 }
335 
336 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
337  SDValue LHS = GetScalarizedVector(N->getOperand(2));
338  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
339  N->getOperand(0), N->getOperand(1),
340  LHS, GetScalarizedVector(N->getOperand(3)),
341  N->getOperand(4));
342 }
343 
344 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
345  assert(N->getValueType(0).isVector() ==
346  N->getOperand(0).getValueType().isVector() &&
347  "Scalar/Vector type mismatch");
348 
349  if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N);
350 
351  SDValue LHS = GetScalarizedVector(N->getOperand(0));
352  SDValue RHS = GetScalarizedVector(N->getOperand(1));
353  SDLoc DL(N);
354 
355  // Turn it into a scalar SETCC.
356  return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2));
357 }
358 
359 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
360  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
361 }
362 
363 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
364  // Figure out if the scalar is the LHS or RHS and return it.
365  SDValue Arg = N->getOperand(2).getOperand(0);
366  if (Arg.getOpcode() == ISD::UNDEF)
367  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
368  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
369  return GetScalarizedVector(N->getOperand(Op));
370 }
371 
372 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
373  assert(N->getValueType(0).isVector() &&
374  N->getOperand(0).getValueType().isVector() &&
375  "Operand types must be vectors");
376  SDValue LHS = N->getOperand(0);
377  SDValue RHS = N->getOperand(1);
378  EVT OpVT = LHS.getValueType();
379  EVT NVT = N->getValueType(0).getVectorElementType();
380  SDLoc DL(N);
381 
382  // The result needs scalarizing, but it's not a given that the source does.
383  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
384  LHS = GetScalarizedVector(LHS);
385  RHS = GetScalarizedVector(RHS);
386  } else {
387  EVT VT = OpVT.getVectorElementType();
388  LHS = DAG.getNode(
389  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
390  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
391  RHS = DAG.getNode(
392  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
393  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
394  }
395 
396  // Turn it into a scalar SETCC.
397  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
398  N->getOperand(2));
399  // Vectors may have a different boolean contents to scalars. Promote the
400  // value appropriately.
401  ISD::NodeType ExtendCode =
403  return DAG.getNode(ExtendCode, DL, NVT, Res);
404 }
405 
406 
407 //===----------------------------------------------------------------------===//
408 // Operand Vector Scalarization <1 x ty> -> ty.
409 //===----------------------------------------------------------------------===//
410 
411 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
412  DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
413  N->dump(&DAG);
414  dbgs() << "\n");
415  SDValue Res = SDValue();
416 
417  if (!Res.getNode()) {
418  switch (N->getOpcode()) {
419  default:
420 #ifndef NDEBUG
421  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
422  N->dump(&DAG);
423  dbgs() << "\n";
424 #endif
425  llvm_unreachable("Do not know how to scalarize this operator's operand!");
426  case ISD::BITCAST:
427  Res = ScalarizeVecOp_BITCAST(N);
428  break;
429  case ISD::ANY_EXTEND:
430  case ISD::ZERO_EXTEND:
431  case ISD::SIGN_EXTEND:
432  case ISD::TRUNCATE:
433  case ISD::FP_TO_SINT:
434  case ISD::FP_TO_UINT:
435  case ISD::SINT_TO_FP:
436  case ISD::UINT_TO_FP:
437  Res = ScalarizeVecOp_UnaryOp(N);
438  break;
439  case ISD::CONCAT_VECTORS:
440  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
441  break;
443  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
444  break;
445  case ISD::VSELECT:
446  Res = ScalarizeVecOp_VSELECT(N);
447  break;
448  case ISD::STORE:
449  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
450  break;
451  case ISD::FP_ROUND:
452  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
453  break;
454  }
455  }
456 
457  // If the result is null, the sub-method took care of registering results etc.
458  if (!Res.getNode()) return false;
459 
460  // If the result is N, the sub-method updated N in place. Tell the legalizer
461  // core about this.
462  if (Res.getNode() == N)
463  return true;
464 
465  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
466  "Invalid operand expansion");
467 
468  ReplaceValueWith(SDValue(N, 0), Res);
469  return false;
470 }
471 
472 /// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs
473 /// to be scalarized, it must be <1 x ty>. Convert the element instead.
474 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
475  SDValue Elt = GetScalarizedVector(N->getOperand(0));
476  return DAG.getNode(ISD::BITCAST, SDLoc(N),
477  N->getValueType(0), Elt);
478 }
479 
480 /// ScalarizeVecOp_UnaryOp - If the input is a vector that needs to be
481 /// scalarized, it must be <1 x ty>. Do the operation on the element instead.
482 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
483  assert(N->getValueType(0).getVectorNumElements() == 1 &&
484  "Unexpected vector type!");
485  SDValue Elt = GetScalarizedVector(N->getOperand(0));
486  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
487  N->getValueType(0).getScalarType(), Elt);
488  // Revectorize the result so the types line up with what the uses of this
489  // expression expect.
490  return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Op);
491 }
492 
493 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one -
494 /// use a BUILD_VECTOR instead.
495 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
497  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
498  Ops[i] = GetScalarizedVector(N->getOperand(i));
499  return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Ops);
500 }
501 
502 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to
503 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the
504 /// index.
505 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
506  SDValue Res = GetScalarizedVector(N->getOperand(0));
507  if (Res.getValueType() != N->getValueType(0))
508  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0),
509  Res);
510  return Res;
511 }
512 
513 
514 /// ScalarizeVecOp_VSELECT - If the input condition is a vector that needs to be
515 /// scalarized, it must be <1 x i1>, so just convert to a normal ISD::SELECT
516 /// (still with vector output type since that was acceptable if we got here).
517 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
518  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
519  EVT VT = N->getValueType(0);
520 
521  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
522  N->getOperand(2));
523 }
524 
525 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
526 /// scalarized, it must be <1 x ty>. Just store the element.
527 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
528  assert(N->isUnindexed() && "Indexed store of one-element vector?");
529  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
530  SDLoc dl(N);
531 
532  if (N->isTruncatingStore())
533  return DAG.getTruncStore(N->getChain(), dl,
534  GetScalarizedVector(N->getOperand(1)),
535  N->getBasePtr(), N->getPointerInfo(),
537  N->isVolatile(), N->isNonTemporal(),
538  N->getAlignment(), N->getAAInfo());
539 
540  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
541  N->getBasePtr(), N->getPointerInfo(),
542  N->isVolatile(), N->isNonTemporal(),
543  N->getOriginalAlignment(), N->getAAInfo());
544 }
545 
546 /// ScalarizeVecOp_FP_ROUND - If the value to round is a vector that needs
547 /// to be scalarized, it must be <1 x ty>. Convert the element instead.
548 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
549  SDValue Elt = GetScalarizedVector(N->getOperand(0));
550  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
551  N->getValueType(0).getVectorElementType(), Elt,
552  N->getOperand(1));
553  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
554 }
555 
556 //===----------------------------------------------------------------------===//
557 // Result Vector Splitting
558 //===----------------------------------------------------------------------===//
559 
560 /// SplitVectorResult - This method is called when the specified result of the
561 /// specified node is found to need vector splitting. At this point, the node
562 /// may also have invalid operands or may have other results that need
563 /// legalization, we just know that (at least) one result needs vector
564 /// splitting.
565 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
566  DEBUG(dbgs() << "Split node result: ";
567  N->dump(&DAG);
568  dbgs() << "\n");
569  SDValue Lo, Hi;
570 
571  // See if the target wants to custom expand this node.
572  if (CustomLowerNode(N, N->getValueType(ResNo), true))
573  return;
574 
575  switch (N->getOpcode()) {
576  default:
577 #ifndef NDEBUG
578  dbgs() << "SplitVectorResult #" << ResNo << ": ";
579  N->dump(&DAG);
580  dbgs() << "\n";
581 #endif
582  report_fatal_error("Do not know how to split the result of this "
583  "operator!\n");
584 
585  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
586  case ISD::VSELECT:
587  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
588  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
589  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
590  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
591  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
592  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
593  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
594  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
595  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
596  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
597  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
598  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
599  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
600  case ISD::LOAD:
601  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
602  break;
603  case ISD::MLOAD:
604  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
605  break;
606  case ISD::MGATHER:
607  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
608  break;
609  case ISD::SETCC:
610  SplitVecRes_SETCC(N, Lo, Hi);
611  break;
612  case ISD::VECTOR_SHUFFLE:
613  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
614  break;
615 
616  case ISD::BSWAP:
617  case ISD::CONVERT_RNDSAT:
618  case ISD::CTLZ:
619  case ISD::CTTZ:
622  case ISD::CTPOP:
623  case ISD::FABS:
624  case ISD::FCEIL:
625  case ISD::FCOS:
626  case ISD::FEXP:
627  case ISD::FEXP2:
628  case ISD::FFLOOR:
629  case ISD::FLOG:
630  case ISD::FLOG10:
631  case ISD::FLOG2:
632  case ISD::FNEARBYINT:
633  case ISD::FNEG:
634  case ISD::FP_EXTEND:
635  case ISD::FP_ROUND:
636  case ISD::FP_TO_SINT:
637  case ISD::FP_TO_UINT:
638  case ISD::FRINT:
639  case ISD::FROUND:
640  case ISD::FSIN:
641  case ISD::FSQRT:
642  case ISD::FTRUNC:
643  case ISD::SINT_TO_FP:
644  case ISD::TRUNCATE:
645  case ISD::UINT_TO_FP:
646  SplitVecRes_UnaryOp(N, Lo, Hi);
647  break;
648 
649  case ISD::ANY_EXTEND:
650  case ISD::SIGN_EXTEND:
651  case ISD::ZERO_EXTEND:
652  SplitVecRes_ExtendOp(N, Lo, Hi);
653  break;
654 
655  case ISD::ADD:
656  case ISD::SUB:
657  case ISD::MUL:
658  case ISD::FADD:
659  case ISD::FCOPYSIGN:
660  case ISD::FSUB:
661  case ISD::FMUL:
662  case ISD::FMINNUM:
663  case ISD::FMAXNUM:
664  case ISD::SDIV:
665  case ISD::UDIV:
666  case ISD::FDIV:
667  case ISD::FPOW:
668  case ISD::AND:
669  case ISD::OR:
670  case ISD::XOR:
671  case ISD::SHL:
672  case ISD::SRA:
673  case ISD::SRL:
674  case ISD::UREM:
675  case ISD::SREM:
676  case ISD::FREM:
677  case ISD::SMIN:
678  case ISD::SMAX:
679  case ISD::UMIN:
680  case ISD::UMAX:
681  SplitVecRes_BinOp(N, Lo, Hi);
682  break;
683  case ISD::FMA:
684  SplitVecRes_TernaryOp(N, Lo, Hi);
685  break;
686  }
687 
688  // If Lo/Hi is null, the sub-method took care of registering results etc.
689  if (Lo.getNode())
690  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
691 }
692 
693 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
694  SDValue &Hi) {
695  SDValue LHSLo, LHSHi;
696  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
697  SDValue RHSLo, RHSHi;
698  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
699  SDLoc dl(N);
700 
701  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo);
702  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi);
703 }
704 
705 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
706  SDValue &Hi) {
707  SDValue Op0Lo, Op0Hi;
708  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
709  SDValue Op1Lo, Op1Hi;
710  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
711  SDValue Op2Lo, Op2Hi;
712  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
713  SDLoc dl(N);
714 
715  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
716  Op0Lo, Op1Lo, Op2Lo);
717  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
718  Op0Hi, Op1Hi, Op2Hi);
719 }
720 
721 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
722  SDValue &Hi) {
723  // We know the result is a vector. The input may be either a vector or a
724  // scalar value.
725  EVT LoVT, HiVT;
726  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
727  SDLoc dl(N);
728 
729  SDValue InOp = N->getOperand(0);
730  EVT InVT = InOp.getValueType();
731 
732  // Handle some special cases efficiently.
733  switch (getTypeAction(InVT)) {
740  break;
743  // A scalar to vector conversion, where the scalar needs expansion.
744  // If the vector is being split in two then we can just convert the
745  // expanded pieces.
746  if (LoVT == HiVT) {
747  GetExpandedOp(InOp, Lo, Hi);
748  if (DAG.getDataLayout().isBigEndian())
749  std::swap(Lo, Hi);
750  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
751  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
752  return;
753  }
754  break;
756  // If the input is a vector that needs to be split, convert each split
757  // piece of the input now.
758  GetSplitVector(InOp, Lo, Hi);
759  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
760  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
761  return;
762  }
763 
764  // In the general case, convert the input to an integer and split it by hand.
765  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
766  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
767  if (DAG.getDataLayout().isBigEndian())
768  std::swap(LoIntVT, HiIntVT);
769 
770  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
771 
772  if (DAG.getDataLayout().isBigEndian())
773  std::swap(Lo, Hi);
774  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
775  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
776 }
777 
778 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
779  SDValue &Hi) {
780  EVT LoVT, HiVT;
781  SDLoc dl(N);
782  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
783  unsigned LoNumElts = LoVT.getVectorNumElements();
784  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
785  Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, LoOps);
786 
787  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
788  Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, HiOps);
789 }
790 
791 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
792  SDValue &Hi) {
793  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
794  SDLoc dl(N);
795  unsigned NumSubvectors = N->getNumOperands() / 2;
796  if (NumSubvectors == 1) {
797  Lo = N->getOperand(0);
798  Hi = N->getOperand(1);
799  return;
800  }
801 
802  EVT LoVT, HiVT;
803  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
804 
805  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
806  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
807 
808  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
809  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
810 }
811 
812 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
813  SDValue &Hi) {
814  SDValue Vec = N->getOperand(0);
815  SDValue Idx = N->getOperand(1);
816  SDLoc dl(N);
817 
818  EVT LoVT, HiVT;
819  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
820 
821  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
822  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
823  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
824  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
825  TLI.getVectorIdxTy(DAG.getDataLayout())));
826 }
827 
828 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
829  SDValue &Hi) {
830  SDValue Vec = N->getOperand(0);
831  SDValue SubVec = N->getOperand(1);
832  SDValue Idx = N->getOperand(2);
833  SDLoc dl(N);
834  GetSplitVector(Vec, Lo, Hi);
835 
836  // Spill the vector to the stack.
837  EVT VecVT = Vec.getValueType();
838  EVT SubVecVT = VecVT.getVectorElementType();
839  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
840  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
841  MachinePointerInfo(), false, false, 0);
842 
843  // Store the new subvector into the specified index.
844  SDValue SubVecPtr = GetVectorElementPointer(StackPtr, SubVecVT, Idx);
845  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
846  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
847  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo(),
848  false, false, 0);
849 
850  // Load the Lo part from the stack slot.
851  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
852  false, false, false, 0);
853 
854  // Increment the pointer to the other part.
855  unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
856  StackPtr =
857  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
858  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
859 
860  // Load the Hi part from the stack slot.
861  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
862  false, false, false, MinAlign(Alignment, IncrementSize));
863 }
864 
865 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
866  SDValue &Hi) {
867  SDLoc dl(N);
868  GetSplitVector(N->getOperand(0), Lo, Hi);
869  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
870  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
871 }
872 
873 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
874  SDValue &Hi) {
875  SDValue LHSLo, LHSHi;
876  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
877  SDLoc dl(N);
878 
879  EVT LoVT, HiVT;
880  std::tie(LoVT, HiVT) =
881  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
882 
883  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
884  DAG.getValueType(LoVT));
885  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
886  DAG.getValueType(HiVT));
887 }
888 
889 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
890  SDValue &Hi) {
891  SDValue Vec = N->getOperand(0);
892  SDValue Elt = N->getOperand(1);
893  SDValue Idx = N->getOperand(2);
894  SDLoc dl(N);
895  GetSplitVector(Vec, Lo, Hi);
896 
897  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
898  unsigned IdxVal = CIdx->getZExtValue();
899  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
900  if (IdxVal < LoNumElts)
901  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
902  Lo.getValueType(), Lo, Elt, Idx);
903  else
904  Hi =
905  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
906  DAG.getConstant(IdxVal - LoNumElts, dl,
907  TLI.getVectorIdxTy(DAG.getDataLayout())));
908  return;
909  }
910 
911  // See if the target wants to custom expand this node.
912  if (CustomLowerNode(N, N->getValueType(0), true))
913  return;
914 
915  // Spill the vector to the stack.
916  EVT VecVT = Vec.getValueType();
917  EVT EltVT = VecVT.getVectorElementType();
918  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
919  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
920  MachinePointerInfo(), false, false, 0);
921 
922  // Store the new element. This may be larger than the vector element type,
923  // so use a truncating store.
924  SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
925  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
926  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
927  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT,
928  false, false, 0);
929 
930  // Load the Lo part from the stack slot.
931  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
932  false, false, false, 0);
933 
934  // Increment the pointer to the other part.
935  unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
936  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
937  DAG.getConstant(IncrementSize, dl,
938  StackPtr.getValueType()));
939 
940  // Load the Hi part from the stack slot.
941  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
942  false, false, false, MinAlign(Alignment, IncrementSize));
943 }
944 
945 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
946  SDValue &Hi) {
947  EVT LoVT, HiVT;
948  SDLoc dl(N);
949  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
950  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
951  Hi = DAG.getUNDEF(HiVT);
952 }
953 
954 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
955  SDValue &Hi) {
956  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
957  EVT LoVT, HiVT;
958  SDLoc dl(LD);
959  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
960 
961  ISD::LoadExtType ExtType = LD->getExtensionType();
962  SDValue Ch = LD->getChain();
963  SDValue Ptr = LD->getBasePtr();
964  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
965  EVT MemoryVT = LD->getMemoryVT();
966  unsigned Alignment = LD->getOriginalAlignment();
967  bool isVolatile = LD->isVolatile();
968  bool isNonTemporal = LD->isNonTemporal();
969  bool isInvariant = LD->isInvariant();
970  AAMDNodes AAInfo = LD->getAAInfo();
971 
972  EVT LoMemVT, HiMemVT;
973  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
974 
975  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
976  LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal,
977  isInvariant, Alignment, AAInfo);
978 
979  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
980  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
981  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
982  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
983  LD->getPointerInfo().getWithOffset(IncrementSize),
984  HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment,
985  AAInfo);
986 
987  // Build a factor node to remember that this load is independent of the
988  // other one.
989  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
990  Hi.getValue(1));
991 
992  // Legalized the chain result - switch anything that used the old chain to
993  // use the new one.
994  ReplaceValueWith(SDValue(LD, 1), Ch);
995 }
996 
997 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
998  SDValue &Lo, SDValue &Hi) {
999  EVT LoVT, HiVT;
1000  SDLoc dl(MLD);
1001  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1002 
1003  SDValue Ch = MLD->getChain();
1004  SDValue Ptr = MLD->getBasePtr();
1005  SDValue Mask = MLD->getMask();
1006  unsigned Alignment = MLD->getOriginalAlignment();
1007  ISD::LoadExtType ExtType = MLD->getExtensionType();
1008 
1009  // if Alignment is equal to the vector size,
1010  // take the half of it for the second part
1011  unsigned SecondHalfAlignment =
1012  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1013  Alignment/2 : Alignment;
1014 
1015  SDValue MaskLo, MaskHi;
1016  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1017 
1018  EVT MemoryVT = MLD->getMemoryVT();
1019  EVT LoMemVT, HiMemVT;
1020  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1021 
1022  SDValue Src0 = MLD->getSrc0();
1023  SDValue Src0Lo, Src0Hi;
1024  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl);
1025 
1027  getMachineMemOperand(MLD->getPointerInfo(),
1029  Alignment, MLD->getAAInfo(), MLD->getRanges());
1030 
1031  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
1032  ExtType);
1033 
1034  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1035  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1036  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
1037 
1038  MMO = DAG.getMachineFunction().
1039  getMachineMemOperand(MLD->getPointerInfo(),
1041  SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());
1042 
1043  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
1044  ExtType);
1045 
1046 
1047  // Build a factor node to remember that this load is independent of the
1048  // other one.
1049  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1050  Hi.getValue(1));
1051 
1052  // Legalized the chain result - switch anything that used the old chain to
1053  // use the new one.
1054  ReplaceValueWith(SDValue(MLD, 1), Ch);
1055 
1056 }
1057 
1058 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1059  SDValue &Lo, SDValue &Hi) {
1060  EVT LoVT, HiVT;
1061  SDLoc dl(MGT);
1062  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1063 
1064  SDValue Ch = MGT->getChain();
1065  SDValue Ptr = MGT->getBasePtr();
1066  SDValue Mask = MGT->getMask();
1067  unsigned Alignment = MGT->getOriginalAlignment();
1068 
1069  SDValue MaskLo, MaskHi;
1070  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1071 
1072  EVT MemoryVT = MGT->getMemoryVT();
1073  EVT LoMemVT, HiMemVT;
1074  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1075 
1076  SDValue Src0Lo, Src0Hi;
1077  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(MGT->getValue(), dl);
1078 
1079  SDValue IndexHi, IndexLo;
1080  std::tie(IndexLo, IndexHi) = DAG.SplitVector(MGT->getIndex(), dl);
1081 
1083  getMachineMemOperand(MGT->getPointerInfo(),
1085  Alignment, MGT->getAAInfo(), MGT->getRanges());
1086 
1087  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
1088  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1089  MMO);
1090 
1091  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
1092  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1093  MMO);
1094 
1095  // Build a factor node to remember that this load is independent of the
1096  // other one.
1097  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1098  Hi.getValue(1));
1099 
1100  // Legalized the chain result - switch anything that used the old chain to
1101  // use the new one.
1102  ReplaceValueWith(SDValue(MGT, 1), Ch);
1103 }
1104 
1105 
1106 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1107  assert(N->getValueType(0).isVector() &&
1108  N->getOperand(0).getValueType().isVector() &&
1109  "Operand types must be vectors");
1110 
1111  EVT LoVT, HiVT;
1112  SDLoc DL(N);
1113  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1114 
1115  // Split the input.
1116  SDValue LL, LH, RL, RH;
1117  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1118  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1119 
1120  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1121  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1122 }
1123 
1124 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1125  SDValue &Hi) {
1126  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1127  EVT LoVT, HiVT;
1128  SDLoc dl(N);
1129  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1130 
1131  // If the input also splits, handle it directly for a compile time speedup.
1132  // Otherwise split it by hand.
1133  EVT InVT = N->getOperand(0).getValueType();
1134  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1135  GetSplitVector(N->getOperand(0), Lo, Hi);
1136  else
1137  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1138 
1139  if (N->getOpcode() == ISD::FP_ROUND) {
1140  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1141  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1142  } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) {
1143  SDValue DTyOpLo = DAG.getValueType(LoVT);
1144  SDValue DTyOpHi = DAG.getValueType(HiVT);
1145  SDValue STyOpLo = DAG.getValueType(Lo.getValueType());
1146  SDValue STyOpHi = DAG.getValueType(Hi.getValueType());
1147  SDValue RndOp = N->getOperand(3);
1148  SDValue SatOp = N->getOperand(4);
1149  ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
1150  Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp,
1151  CvtCode);
1152  Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp,
1153  CvtCode);
1154  } else {
1155  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1156  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1157  }
1158 }
1159 
1160 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1161  SDValue &Hi) {
1162  SDLoc dl(N);
1163  EVT SrcVT = N->getOperand(0).getValueType();
1164  EVT DestVT = N->getValueType(0);
1165  EVT LoVT, HiVT;
1166  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1167 
1168  // We can do better than a generic split operation if the extend is doing
1169  // more than just doubling the width of the elements and the following are
1170  // true:
1171  // - The number of vector elements is even,
1172  // - the source type is legal,
1173  // - the type of a split source is illegal,
1174  // - the type of an extended (by doubling element size) source is legal, and
1175  // - the type of that extended source when split is legal.
1176  //
1177  // This won't necessarily completely legalize the operation, but it will
1178  // more effectively move in the right direction and prevent falling down
1179  // to scalarization in many cases due to the input vector being split too
1180  // far.
1181  unsigned NumElements = SrcVT.getVectorNumElements();
1182  if ((NumElements & 1) == 0 &&
1183  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1184  LLVMContext &Ctx = *DAG.getContext();
1185  EVT NewSrcVT = EVT::getVectorVT(
1186  Ctx, EVT::getIntegerVT(
1187  Ctx, SrcVT.getVectorElementType().getSizeInBits() * 2),
1188  NumElements);
1189  EVT SplitSrcVT =
1190  EVT::getVectorVT(Ctx, SrcVT.getVectorElementType(), NumElements / 2);
1191  EVT SplitLoVT, SplitHiVT;
1192  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1193  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1194  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1195  DEBUG(dbgs() << "Split vector extend via incremental extend:";
1196  N->dump(&DAG); dbgs() << "\n");
1197  // Extend the source vector by one step.
1198  SDValue NewSrc =
1199  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1200  // Get the low and high halves of the new, extended one step, vector.
1201  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1202  // Extend those vector halves the rest of the way.
1203  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1204  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1205  return;
1206  }
1207  }
1208  // Fall back to the generic unary operator splitting otherwise.
1209  SplitVecRes_UnaryOp(N, Lo, Hi);
1210 }
1211 
1212 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1213  SDValue &Lo, SDValue &Hi) {
1214  // The low and high parts of the original input give four input vectors.
1215  SDValue Inputs[4];
1216  SDLoc dl(N);
1217  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1218  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1219  EVT NewVT = Inputs[0].getValueType();
1220  unsigned NewElts = NewVT.getVectorNumElements();
1221 
1222  // If Lo or Hi uses elements from at most two of the four input vectors, then
1223  // express it as a vector shuffle of those two inputs. Otherwise extract the
1224  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1226  for (unsigned High = 0; High < 2; ++High) {
1227  SDValue &Output = High ? Hi : Lo;
1228 
1229  // Build a shuffle mask for the output, discovering on the fly which
1230  // input vectors to use as shuffle operands (recorded in InputUsed).
1231  // If building a suitable shuffle vector proves too hard, then bail
1232  // out with useBuildVector set.
1233  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1234  unsigned FirstMaskIdx = High * NewElts;
1235  bool useBuildVector = false;
1236  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1237  // The mask element. This indexes into the input.
1238  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1239 
1240  // The input vector this mask element indexes into.
1241  unsigned Input = (unsigned)Idx / NewElts;
1242 
1243  if (Input >= array_lengthof(Inputs)) {
1244  // The mask element does not index into any input vector.
1245  Ops.push_back(-1);
1246  continue;
1247  }
1248 
1249  // Turn the index into an offset from the start of the input vector.
1250  Idx -= Input * NewElts;
1251 
1252  // Find or create a shuffle vector operand to hold this input.
1253  unsigned OpNo;
1254  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1255  if (InputUsed[OpNo] == Input) {
1256  // This input vector is already an operand.
1257  break;
1258  } else if (InputUsed[OpNo] == -1U) {
1259  // Create a new operand for this input vector.
1260  InputUsed[OpNo] = Input;
1261  break;
1262  }
1263  }
1264 
1265  if (OpNo >= array_lengthof(InputUsed)) {
1266  // More than two input vectors used! Give up on trying to create a
1267  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1268  useBuildVector = true;
1269  break;
1270  }
1271 
1272  // Add the mask index for the new shuffle vector.
1273  Ops.push_back(Idx + OpNo * NewElts);
1274  }
1275 
1276  if (useBuildVector) {
1277  EVT EltVT = NewVT.getVectorElementType();
1279 
1280  // Extract the input elements by hand.
1281  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1282  // The mask element. This indexes into the input.
1283  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1284 
1285  // The input vector this mask element indexes into.
1286  unsigned Input = (unsigned)Idx / NewElts;
1287 
1288  if (Input >= array_lengthof(Inputs)) {
1289  // The mask element is "undef" or indexes off the end of the input.
1290  SVOps.push_back(DAG.getUNDEF(EltVT));
1291  continue;
1292  }
1293 
1294  // Turn the index into an offset from the start of the input vector.
1295  Idx -= Input * NewElts;
1296 
1297  // Extract the vector element by hand.
1298  SVOps.push_back(DAG.getNode(
1299  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1300  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1301  }
1302 
1303  // Construct the Lo/Hi output using a BUILD_VECTOR.
1304  Output = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, SVOps);
1305  } else if (InputUsed[0] == -1U) {
1306  // No input vectors were used! The result is undefined.
1307  Output = DAG.getUNDEF(NewVT);
1308  } else {
1309  SDValue Op0 = Inputs[InputUsed[0]];
1310  // If only one input was used, use an undefined vector for the other.
1311  SDValue Op1 = InputUsed[1] == -1U ?
1312  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1313  // At least one input vector was used. Create a new shuffle vector.
1314  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]);
1315  }
1316 
1317  Ops.clear();
1318  }
1319 }
1320 
1321 
1322 //===----------------------------------------------------------------------===//
1323 // Operand Vector Splitting
1324 //===----------------------------------------------------------------------===//
1325 
1326 /// SplitVectorOperand - This method is called when the specified operand of the
1327 /// specified node is found to need vector splitting. At this point, all of the
1328 /// result types of the node are known to be legal, but other operands of the
1329 /// node may need legalization as well as the specified one.
1330 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1331  DEBUG(dbgs() << "Split node operand: ";
1332  N->dump(&DAG);
1333  dbgs() << "\n");
1334  SDValue Res = SDValue();
1335 
1336  // See if the target wants to custom split this node.
1337  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1338  return false;
1339 
1340  if (!Res.getNode()) {
1341  switch (N->getOpcode()) {
1342  default:
1343 #ifndef NDEBUG
1344  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1345  N->dump(&DAG);
1346  dbgs() << "\n";
1347 #endif
1348  report_fatal_error("Do not know how to split this operator's "
1349  "operand!\n");
1350 
1351  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1352  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1353  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1354  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1355  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1356  case ISD::TRUNCATE:
1357  Res = SplitVecOp_TruncateHelper(N);
1358  break;
1359  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1360  case ISD::STORE:
1361  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1362  break;
1363  case ISD::MSTORE:
1364  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1365  break;
1366  case ISD::MSCATTER:
1367  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1368  break;
1369  case ISD::MGATHER:
1370  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1371  break;
1372  case ISD::VSELECT:
1373  Res = SplitVecOp_VSELECT(N, OpNo);
1374  break;
1375  case ISD::FP_TO_SINT:
1376  case ISD::FP_TO_UINT:
1377  if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
1378  Res = SplitVecOp_TruncateHelper(N);
1379  else
1380  Res = SplitVecOp_UnaryOp(N);
1381  break;
1382  case ISD::SINT_TO_FP:
1383  case ISD::UINT_TO_FP:
1384  if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0)))
1385  Res = SplitVecOp_TruncateHelper(N);
1386  else
1387  Res = SplitVecOp_UnaryOp(N);
1388  break;
1389  case ISD::CTTZ:
1390  case ISD::CTLZ:
1391  case ISD::CTPOP:
1392  case ISD::FP_EXTEND:
1393  case ISD::SIGN_EXTEND:
1394  case ISD::ZERO_EXTEND:
1395  case ISD::ANY_EXTEND:
1396  case ISD::FTRUNC:
1397  Res = SplitVecOp_UnaryOp(N);
1398  break;
1399  }
1400  }
1401 
1402  // If the result is null, the sub-method took care of registering results etc.
1403  if (!Res.getNode()) return false;
1404 
1405  // If the result is N, the sub-method updated N in place. Tell the legalizer
1406  // core about this.
1407  if (Res.getNode() == N)
1408  return true;
1409 
1410  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1411  "Invalid operand expansion");
1412 
1413  ReplaceValueWith(SDValue(N, 0), Res);
1414  return false;
1415 }
1416 
1417 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1418  // The only possibility for an illegal operand is the mask, since result type
1419  // legalization would have handled this node already otherwise.
1420  assert(OpNo == 0 && "Illegal operand must be mask");
1421 
1422  SDValue Mask = N->getOperand(0);
1423  SDValue Src0 = N->getOperand(1);
1424  SDValue Src1 = N->getOperand(2);
1425  EVT Src0VT = Src0.getValueType();
1426  SDLoc DL(N);
1427  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1428 
1429  SDValue Lo, Hi;
1430  GetSplitVector(N->getOperand(0), Lo, Hi);
1431  assert(Lo.getValueType() == Hi.getValueType() &&
1432  "Lo and Hi have differing types");
1433 
1434  EVT LoOpVT, HiOpVT;
1435  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1436  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1437 
1438  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1439  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1440  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1441  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1442 
1443  SDValue LoSelect =
1444  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1445  SDValue HiSelect =
1446  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1447 
1448  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1449 }
1450 
1451 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1452  // The result has a legal vector type, but the input needs splitting.
1453  EVT ResVT = N->getValueType(0);
1454  SDValue Lo, Hi;
1455  SDLoc dl(N);
1456  GetSplitVector(N->getOperand(0), Lo, Hi);
1457  EVT InVT = Lo.getValueType();
1458 
1459  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1460  InVT.getVectorNumElements());
1461 
1462  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1463  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1464 
1465  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1466 }
1467 
1468 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1469  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1470  // end up being split all the way down to individual components. Convert the
1471  // split pieces into integers and reassemble.
1472  SDValue Lo, Hi;
1473  GetSplitVector(N->getOperand(0), Lo, Hi);
1474  Lo = BitConvertToInteger(Lo);
1475  Hi = BitConvertToInteger(Hi);
1476 
1477  if (DAG.getDataLayout().isBigEndian())
1478  std::swap(Lo, Hi);
1479 
1480  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1481  JoinIntegers(Lo, Hi));
1482 }
1483 
1484 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1485  // We know that the extracted result type is legal.
1486  EVT SubVT = N->getValueType(0);
1487  SDValue Idx = N->getOperand(1);
1488  SDLoc dl(N);
1489  SDValue Lo, Hi;
1490  GetSplitVector(N->getOperand(0), Lo, Hi);
1491 
1492  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1493  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1494 
1495  if (IdxVal < LoElts) {
1496  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1497  "Extracted subvector crosses vector split!");
1498  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1499  } else {
1500  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1501  DAG.getConstant(IdxVal - LoElts, dl,
1502  Idx.getValueType()));
1503  }
1504 }
1505 
1506 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1507  SDValue Vec = N->getOperand(0);
1508  SDValue Idx = N->getOperand(1);
1509  EVT VecVT = Vec.getValueType();
1510 
1511  if (isa<ConstantSDNode>(Idx)) {
1512  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1513  assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1514 
1515  SDValue Lo, Hi;
1516  GetSplitVector(Vec, Lo, Hi);
1517 
1518  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1519 
1520  if (IdxVal < LoElts)
1521  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1522  return SDValue(DAG.UpdateNodeOperands(N, Hi,
1523  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
1524  Idx.getValueType())), 0);
1525  }
1526 
1527  // See if the target wants to custom expand this node.
1528  if (CustomLowerNode(N, N->getValueType(0), true))
1529  return SDValue();
1530 
1531  // Store the vector to the stack.
1532  EVT EltVT = VecVT.getVectorElementType();
1533  SDLoc dl(N);
1534  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1535  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1536  MachinePointerInfo(), false, false, 0);
1537 
1538  // Load back the required element.
1539  StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
1540  return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1541  MachinePointerInfo(), EltVT, false, false, false, 0);
1542 }
1543 
1544 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
1545  unsigned OpNo) {
1546  EVT LoVT, HiVT;
1547  SDLoc dl(MGT);
1548  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1549 
1550  SDValue Ch = MGT->getChain();
1551  SDValue Ptr = MGT->getBasePtr();
1552  SDValue Index = MGT->getIndex();
1553  SDValue Mask = MGT->getMask();
1554  unsigned Alignment = MGT->getOriginalAlignment();
1555 
1556  SDValue MaskLo, MaskHi;
1557  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1558 
1559  EVT MemoryVT = MGT->getMemoryVT();
1560  EVT LoMemVT, HiMemVT;
1561  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1562 
1563  SDValue Src0Lo, Src0Hi;
1564  std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(MGT->getValue(), dl);
1565 
1566  SDValue IndexHi, IndexLo;
1567  if (Index.getNode())
1568  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1569  else
1570  IndexLo = IndexHi = Index;
1571 
1573  getMachineMemOperand(MGT->getPointerInfo(),
1575  Alignment, MGT->getAAInfo(), MGT->getRanges());
1576 
1577  SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo};
1578  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
1579  OpsLo, MMO);
1580 
1581  MMO = DAG.getMachineFunction().
1582  getMachineMemOperand(MGT->getPointerInfo(),
1584  Alignment, MGT->getAAInfo(),
1585  MGT->getRanges());
1586 
1587  SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi};
1588  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
1589  OpsHi, MMO);
1590 
1591  // Build a factor node to remember that this load is independent of the
1592  // other one.
1593  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1594  Hi.getValue(1));
1595 
1596  // Legalized the chain result - switch anything that used the old chain to
1597  // use the new one.
1598  ReplaceValueWith(SDValue(MGT, 1), Ch);
1599 
1600  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
1601  Hi);
1602  ReplaceValueWith(SDValue(MGT, 0), Res);
1603  return SDValue();
1604 }
1605 
1606 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
1607  unsigned OpNo) {
1608  SDValue Ch = N->getChain();
1609  SDValue Ptr = N->getBasePtr();
1610  SDValue Mask = N->getMask();
1611  SDValue Data = N->getValue();
1612  EVT MemoryVT = N->getMemoryVT();
1613  unsigned Alignment = N->getOriginalAlignment();
1614  SDLoc DL(N);
1615 
1616  EVT LoMemVT, HiMemVT;
1617  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1618 
1619  SDValue DataLo, DataHi;
1620  GetSplitVector(Data, DataLo, DataHi);
1621  SDValue MaskLo, MaskHi;
1622  GetSplitVector(Mask, MaskLo, MaskHi);
1623 
1624  // if Alignment is equal to the vector size,
1625  // take the half of it for the second part
1626  unsigned SecondHalfAlignment =
1627  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
1628  Alignment/2 : Alignment;
1629 
1630  SDValue Lo, Hi;
1632  getMachineMemOperand(N->getPointerInfo(),
1634  Alignment, N->getAAInfo(), N->getRanges());
1635 
1636  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
1637  N->isTruncatingStore());
1638 
1639  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1640  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
1641  DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
1642 
1643  MMO = DAG.getMachineFunction().
1644  getMachineMemOperand(N->getPointerInfo(),
1646  SecondHalfAlignment, N->getAAInfo(), N->getRanges());
1647 
1648  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
1649  N->isTruncatingStore());
1650 
1651  // Build a factor node to remember that this store is independent of the
1652  // other one.
1653  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1654 }
1655 
1656 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
1657  unsigned OpNo) {
1658  SDValue Ch = N->getChain();
1659  SDValue Ptr = N->getBasePtr();
1660  SDValue Mask = N->getMask();
1661  SDValue Index = N->getIndex();
1662  SDValue Data = N->getValue();
1663  EVT MemoryVT = N->getMemoryVT();
1664  unsigned Alignment = N->getOriginalAlignment();
1665  SDLoc DL(N);
1666 
1667  EVT LoMemVT, HiMemVT;
1668  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1669 
1670  SDValue DataLo, DataHi;
1671  GetSplitVector(Data, DataLo, DataHi);
1672  SDValue MaskLo, MaskHi;
1673  GetSplitVector(Mask, MaskLo, MaskHi);
1674 
1675  SDValue PtrLo, PtrHi;
1676  if (Ptr.getValueType().isVector()) // gather form vector of pointers
1677  std::tie(PtrLo, PtrHi) = DAG.SplitVector(Ptr, DL);
1678  else
1679  PtrLo = PtrHi = Ptr;
1680 
1681  SDValue IndexHi, IndexLo;
1682  if (Index.getNode())
1683  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
1684  else
1685  IndexLo = IndexHi = Index;
1686 
1687  SDValue Lo, Hi;
1689  getMachineMemOperand(N->getPointerInfo(),
1691  Alignment, N->getAAInfo(), N->getRanges());
1692 
1693  SDValue OpsLo[] = {Ch, DataLo, MaskLo, PtrLo, IndexLo};
1694  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
1695  DL, OpsLo, MMO);
1696 
1697  MMO = DAG.getMachineFunction().
1698  getMachineMemOperand(N->getPointerInfo(),
1700  Alignment, N->getAAInfo(), N->getRanges());
1701 
1702  SDValue OpsHi[] = {Ch, DataHi, MaskHi, PtrHi, IndexHi};
1703  Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
1704  DL, OpsHi, MMO);
1705 
1706  // Build a factor node to remember that this store is independent of the
1707  // other one.
1708  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1709 }
1710 
1711 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
1712  assert(N->isUnindexed() && "Indexed store of vector?");
1713  assert(OpNo == 1 && "Can only split the stored value");
1714  SDLoc DL(N);
1715 
1716  bool isTruncating = N->isTruncatingStore();
1717  SDValue Ch = N->getChain();
1718  SDValue Ptr = N->getBasePtr();
1719  EVT MemoryVT = N->getMemoryVT();
1720  unsigned Alignment = N->getOriginalAlignment();
1721  bool isVol = N->isVolatile();
1722  bool isNT = N->isNonTemporal();
1723  AAMDNodes AAInfo = N->getAAInfo();
1724  SDValue Lo, Hi;
1725  GetSplitVector(N->getOperand(1), Lo, Hi);
1726 
1727  EVT LoMemVT, HiMemVT;
1728  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1729 
1730  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1731 
1732  if (isTruncating)
1733  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(),
1734  LoMemVT, isVol, isNT, Alignment, AAInfo);
1735  else
1736  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(),
1737  isVol, isNT, Alignment, AAInfo);
1738 
1739  // Increment the pointer to the other half.
1740  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
1741  DAG.getConstant(IncrementSize, DL, Ptr.getValueType()));
1742 
1743  if (isTruncating)
1744  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
1745  N->getPointerInfo().getWithOffset(IncrementSize),
1746  HiMemVT, isVol, isNT, Alignment, AAInfo);
1747  else
1748  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
1749  N->getPointerInfo().getWithOffset(IncrementSize),
1750  isVol, isNT, Alignment, AAInfo);
1751 
1752  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1753 }
1754 
1755 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
1756  SDLoc DL(N);
1757 
1758  // The input operands all must have the same type, and we know the result
1759  // type is valid. Convert this to a buildvector which extracts all the
1760  // input elements.
1761  // TODO: If the input elements are power-two vectors, we could convert this to
1762  // a new CONCAT_VECTORS node with elements that are half-wide.
1764  EVT EltVT = N->getValueType(0).getVectorElementType();
1765  for (const SDValue &Op : N->op_values()) {
1766  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
1767  i != e; ++i) {
1768  Elts.push_back(DAG.getNode(
1769  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
1770  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1771  }
1772  }
1773 
1774  return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts);
1775 }
1776 
1777 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
1778  // The result type is legal, but the input type is illegal. If splitting
1779  // ends up with the result type of each half still being legal, just
1780  // do that. If, however, that would result in an illegal result type,
1781  // we can try to get more clever with power-two vectors. Specifically,
1782  // split the input type, but also widen the result element size, then
1783  // concatenate the halves and truncate again. For example, consider a target
1784  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
1785  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
1786  // %inlo = v4i32 extract_subvector %in, 0
1787  // %inhi = v4i32 extract_subvector %in, 4
1788  // %lo16 = v4i16 trunc v4i32 %inlo
1789  // %hi16 = v4i16 trunc v4i32 %inhi
1790  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
1791  // %res = v8i8 trunc v8i16 %in16
1792  //
1793  // Without this transform, the original truncate would end up being
1794  // scalarized, which is pretty much always a last resort.
1795  SDValue InVec = N->getOperand(0);
1796  EVT InVT = InVec->getValueType(0);
1797  EVT OutVT = N->getValueType(0);
1798  unsigned NumElements = OutVT.getVectorNumElements();
1799  bool IsFloat = OutVT.isFloatingPoint();
1800 
1801  // Widening should have already made sure this is a power-two vector
1802  // if we're trying to split it at all. assert() that's true, just in case.
1803  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
1804 
1805  unsigned InElementSize = InVT.getVectorElementType().getSizeInBits();
1806  unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits();
1807 
1808  // If the input elements are only 1/2 the width of the result elements,
1809  // just use the normal splitting. Our trick only work if there's room
1810  // to split more than once.
1811  if (InElementSize <= OutElementSize * 2)
1812  return SplitVecOp_UnaryOp(N);
1813  SDLoc DL(N);
1814 
1815  // Extract the halves of the input via extract_subvector.
1816  SDValue InLoVec, InHiVec;
1817  std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL);
1818  // Truncate them to 1/2 the element size.
1819  EVT HalfElementVT = IsFloat ?
1820  EVT::getFloatingPointVT(InElementSize/2) :
1821  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
1822  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
1823  NumElements/2);
1824  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
1825  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
1826  // Concatenate them to get the full intermediate truncation result.
1827  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
1828  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
1829  HalfHi);
1830  // Now finish up by truncating all the way down to the original result
1831  // type. This should normally be something that ends up being legal directly,
1832  // but in theory if a target has very wide vectors and an annoyingly
1833  // restricted set of legal types, this split can chain to build things up.
1834  return IsFloat
1835  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
1836  DAG.getTargetConstant(
1837  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
1838  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
1839 }
1840 
1841 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
1842  assert(N->getValueType(0).isVector() &&
1843  N->getOperand(0).getValueType().isVector() &&
1844  "Operand types must be vectors");
1845  // The result has a legal vector type, but the input needs splitting.
1846  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
1847  SDLoc DL(N);
1848  GetSplitVector(N->getOperand(0), Lo0, Hi0);
1849  GetSplitVector(N->getOperand(1), Lo1, Hi1);
1850  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
1851  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
1852  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
1853 
1854  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
1855  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
1856  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
1857  return PromoteTargetBoolean(Con, N->getValueType(0));
1858 }
1859 
1860 
1861 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
1862  // The result has a legal vector type, but the input needs splitting.
1863  EVT ResVT = N->getValueType(0);
1864  SDValue Lo, Hi;
1865  SDLoc DL(N);
1866  GetSplitVector(N->getOperand(0), Lo, Hi);
1867  EVT InVT = Lo.getValueType();
1868 
1869  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1870  InVT.getVectorNumElements());
1871 
1872  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
1873  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
1874 
1875  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
1876 }
1877 
1878 
1879 
1880 //===----------------------------------------------------------------------===//
1881 // Result Vector Widening
1882 //===----------------------------------------------------------------------===//
1883 
1884 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
1885  DEBUG(dbgs() << "Widen node result " << ResNo << ": ";
1886  N->dump(&DAG);
1887  dbgs() << "\n");
1888 
1889  // See if the target wants to custom widen this node.
1890  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
1891  return;
1892 
1893  SDValue Res = SDValue();
1894  switch (N->getOpcode()) {
1895  default:
1896 #ifndef NDEBUG
1897  dbgs() << "WidenVectorResult #" << ResNo << ": ";
1898  N->dump(&DAG);
1899  dbgs() << "\n";
1900 #endif
1901  llvm_unreachable("Do not know how to widen the result of this operator!");
1902 
1903  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
1904  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
1905  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
1906  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
1907  case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break;
1908  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
1909  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
1910  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
1911  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
1912  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
1913  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
1914  case ISD::VSELECT:
1915  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
1916  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
1917  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
1918  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
1919  case ISD::VECTOR_SHUFFLE:
1920  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
1921  break;
1922  case ISD::MLOAD:
1923  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
1924  break;
1925 
1926  case ISD::ADD:
1927  case ISD::AND:
1928  case ISD::MUL:
1929  case ISD::MULHS:
1930  case ISD::MULHU:
1931  case ISD::OR:
1932  case ISD::SUB:
1933  case ISD::XOR:
1934  case ISD::FMINNUM:
1935  case ISD::FMAXNUM:
1936  Res = WidenVecRes_Binary(N);
1937  break;
1938 
1939  case ISD::FADD:
1940  case ISD::FCOPYSIGN:
1941  case ISD::FMUL:
1942  case ISD::FPOW:
1943  case ISD::FSUB:
1944  case ISD::FDIV:
1945  case ISD::FREM:
1946  case ISD::SDIV:
1947  case ISD::UDIV:
1948  case ISD::SREM:
1949  case ISD::UREM:
1950  Res = WidenVecRes_BinaryCanTrap(N);
1951  break;
1952 
1953  case ISD::FPOWI:
1954  Res = WidenVecRes_POWI(N);
1955  break;
1956 
1957  case ISD::SHL:
1958  case ISD::SRA:
1959  case ISD::SRL:
1960  Res = WidenVecRes_Shift(N);
1961  break;
1962 
1963  case ISD::ANY_EXTEND:
1964  case ISD::FP_EXTEND:
1965  case ISD::FP_ROUND:
1966  case ISD::FP_TO_SINT:
1967  case ISD::FP_TO_UINT:
1968  case ISD::SIGN_EXTEND:
1969  case ISD::SINT_TO_FP:
1970  case ISD::TRUNCATE:
1971  case ISD::UINT_TO_FP:
1972  case ISD::ZERO_EXTEND:
1973  Res = WidenVecRes_Convert(N);
1974  break;
1975 
1976  case ISD::BSWAP:
1977  case ISD::CTLZ:
1978  case ISD::CTPOP:
1979  case ISD::CTTZ:
1980  case ISD::FABS:
1981  case ISD::FCEIL:
1982  case ISD::FCOS:
1983  case ISD::FEXP:
1984  case ISD::FEXP2:
1985  case ISD::FFLOOR:
1986  case ISD::FLOG:
1987  case ISD::FLOG10:
1988  case ISD::FLOG2:
1989  case ISD::FNEARBYINT:
1990  case ISD::FNEG:
1991  case ISD::FRINT:
1992  case ISD::FROUND:
1993  case ISD::FSIN:
1994  case ISD::FSQRT:
1995  case ISD::FTRUNC:
1996  Res = WidenVecRes_Unary(N);
1997  break;
1998  case ISD::FMA:
1999  Res = WidenVecRes_Ternary(N);
2000  break;
2001  }
2002 
2003  // If Res is null, the sub-method took care of registering the result.
2004  if (Res.getNode())
2005  SetWidenedVector(SDValue(N, ResNo), Res);
2006 }
2007 
2008 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2009  // Ternary op widening.
2010  SDLoc dl(N);
2011  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2012  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2013  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2014  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2015  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2016 }
2017 
2018 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2019  // Binary op widening.
2020  SDLoc dl(N);
2021  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2022  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2023  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2024  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2);
2025 }
2026 
2027 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2028  // Binary op widening for operations that can trap.
2029  unsigned Opcode = N->getOpcode();
2030  SDLoc dl(N);
2031  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2032  EVT WidenEltVT = WidenVT.getVectorElementType();
2033  EVT VT = WidenVT;
2034  unsigned NumElts = VT.getVectorNumElements();
2035  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2036  NumElts = NumElts / 2;
2037  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2038  }
2039 
2040  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2041  // Operation doesn't trap so just widen as normal.
2042  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2043  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2044  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2);
2045  }
2046 
2047  // No legal vector version so unroll the vector operation and then widen.
2048  if (NumElts == 1)
2049  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2050 
2051  // Since the operation can trap, apply operation on the original vector.
2052  EVT MaxVT = VT;
2053  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2054  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2055  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2056 
2057  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2058  unsigned ConcatEnd = 0; // Current ConcatOps index.
2059  int Idx = 0; // Current Idx into input vectors.
2060 
2061  // NumElts := greatest legal vector size (at most WidenVT)
2062  // while (orig. vector has unhandled elements) {
2063  // take munches of size NumElts from the beginning and add to ConcatOps
2064  // NumElts := next smaller supported vector size or 1
2065  // }
2066  while (CurNumElts != 0) {
2067  while (CurNumElts >= NumElts) {
2068  SDValue EOp1 = DAG.getNode(
2069  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2070  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2071  SDValue EOp2 = DAG.getNode(
2072  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2073  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2074  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2);
2075  Idx += NumElts;
2076  CurNumElts -= NumElts;
2077  }
2078  do {
2079  NumElts = NumElts / 2;
2080  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2081  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2082 
2083  if (NumElts == 1) {
2084  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2085  SDValue EOp1 = DAG.getNode(
2086  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2087  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2088  SDValue EOp2 = DAG.getNode(
2089  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2090  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2091  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2092  EOp1, EOp2);
2093  }
2094  CurNumElts = 0;
2095  }
2096  }
2097 
2098  // Check to see if we have a single operation with the widen type.
2099  if (ConcatEnd == 1) {
2100  VT = ConcatOps[0].getValueType();
2101  if (VT == WidenVT)
2102  return ConcatOps[0];
2103  }
2104 
2105  // while (Some element of ConcatOps is not of type MaxVT) {
2106  // From the end of ConcatOps, collect elements of the same type and put
2107  // them into an op of the next larger supported type
2108  // }
2109  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2110  Idx = ConcatEnd - 1;
2111  VT = ConcatOps[Idx--].getValueType();
2112  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2113  Idx--;
2114 
2115  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2116  EVT NextVT;
2117  do {
2118  NextSize *= 2;
2119  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2120  } while (!TLI.isTypeLegal(NextVT));
2121 
2122  if (!VT.isVector()) {
2123  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2124  SDValue VecOp = DAG.getUNDEF(NextVT);
2125  unsigned NumToInsert = ConcatEnd - Idx - 1;
2126  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2127  VecOp = DAG.getNode(
2128  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2129  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2130  }
2131  ConcatOps[Idx+1] = VecOp;
2132  ConcatEnd = Idx + 2;
2133  } else {
2134  // Vector type, create a CONCAT_VECTORS of type NextVT
2135  SDValue undefVec = DAG.getUNDEF(VT);
2136  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2137  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2138  unsigned RealVals = ConcatEnd - Idx - 1;
2139  unsigned SubConcatEnd = 0;
2140  unsigned SubConcatIdx = Idx + 1;
2141  while (SubConcatEnd < RealVals)
2142  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2143  while (SubConcatEnd < OpsToConcat)
2144  SubConcatOps[SubConcatEnd++] = undefVec;
2145  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2146  NextVT, SubConcatOps);
2147  ConcatEnd = SubConcatIdx + 1;
2148  }
2149  }
2150 
2151  // Check to see if we have a single operation with the widen type.
2152  if (ConcatEnd == 1) {
2153  VT = ConcatOps[0].getValueType();
2154  if (VT == WidenVT)
2155  return ConcatOps[0];
2156  }
2157 
2158  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2159  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2160  if (NumOps != ConcatEnd ) {
2161  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2162  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2163  ConcatOps[j] = UndefVal;
2164  }
2165  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2166  makeArrayRef(ConcatOps.data(), NumOps));
2167 }
2168 
2169 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
2170  SDValue InOp = N->getOperand(0);
2171  SDLoc DL(N);
2172 
2173  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2174  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2175 
2176  EVT InVT = InOp.getValueType();
2177  EVT InEltVT = InVT.getVectorElementType();
2178  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2179 
2180  unsigned Opcode = N->getOpcode();
2181  unsigned InVTNumElts = InVT.getVectorNumElements();
2182 
2183  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2184  InOp = GetWidenedVector(N->getOperand(0));
2185  InVT = InOp.getValueType();
2186  InVTNumElts = InVT.getVectorNumElements();
2187  if (InVTNumElts == WidenNumElts) {
2188  if (N->getNumOperands() == 1)
2189  return DAG.getNode(Opcode, DL, WidenVT, InOp);
2190  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1));
2191  }
2192  }
2193 
2194  if (TLI.isTypeLegal(InWidenVT)) {
2195  // Because the result and the input are different vector types, widening
2196  // the result could create a legal type but widening the input might make
2197  // it an illegal type that might lead to repeatedly splitting the input
2198  // and then widening it. To avoid this, we widen the input only if
2199  // it results in a legal type.
2200  if (WidenNumElts % InVTNumElts == 0) {
2201  // Widen the input and call convert on the widened input vector.
2202  unsigned NumConcat = WidenNumElts/InVTNumElts;
2203  SmallVector<SDValue, 16> Ops(NumConcat);
2204  Ops[0] = InOp;
2205  SDValue UndefVal = DAG.getUNDEF(InVT);
2206  for (unsigned i = 1; i != NumConcat; ++i)
2207  Ops[i] = UndefVal;
2208  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
2209  if (N->getNumOperands() == 1)
2210  return DAG.getNode(Opcode, DL, WidenVT, InVec);
2211  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1));
2212  }
2213 
2214  if (InVTNumElts % WidenNumElts == 0) {
2215  SDValue InVal = DAG.getNode(
2216  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
2217  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2218  // Extract the input and convert the shorten input vector.
2219  if (N->getNumOperands() == 1)
2220  return DAG.getNode(Opcode, DL, WidenVT, InVal);
2221  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1));
2222  }
2223  }
2224 
2225  // Otherwise unroll into some nasty scalar code and rebuild the vector.
2226  SmallVector<SDValue, 16> Ops(WidenNumElts);
2227  EVT EltVT = WidenVT.getVectorElementType();
2228  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2229  unsigned i;
2230  for (i=0; i < MinElts; ++i) {
2231  SDValue Val = DAG.getNode(
2232  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
2233  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2234  if (N->getNumOperands() == 1)
2235  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
2236  else
2237  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1));
2238  }
2239 
2240  SDValue UndefVal = DAG.getUNDEF(EltVT);
2241  for (; i < WidenNumElts; ++i)
2242  Ops[i] = UndefVal;
2243 
2244  return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops);
2245 }
2246 
2247 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
2248  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2249  SDValue InOp = GetWidenedVector(N->getOperand(0));
2250  SDValue ShOp = N->getOperand(1);
2251  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2252 }
2253 
2254 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
2255  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2256  SDValue InOp = GetWidenedVector(N->getOperand(0));
2257  SDValue ShOp = N->getOperand(1);
2258 
2259  EVT ShVT = ShOp.getValueType();
2260  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
2261  ShOp = GetWidenedVector(ShOp);
2262  ShVT = ShOp.getValueType();
2263  }
2264  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
2265  ShVT.getVectorElementType(),
2266  WidenVT.getVectorNumElements());
2267  if (ShVT != ShWidenVT)
2268  ShOp = ModifyToType(ShOp, ShWidenVT);
2269 
2270  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2271 }
2272 
2273 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
2274  // Unary op widening.
2275  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2276  SDValue InOp = GetWidenedVector(N->getOperand(0));
2277  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
2278 }
2279 
2280 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
2281  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2282  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
2283  cast<VTSDNode>(N->getOperand(1))->getVT()
2284  .getVectorElementType(),
2285  WidenVT.getVectorNumElements());
2286  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
2287  return DAG.getNode(N->getOpcode(), SDLoc(N),
2288  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
2289 }
2290 
2291 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
2292  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2293  return GetWidenedVector(WidenVec);
2294 }
2295 
2296 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
2297  SDValue InOp = N->getOperand(0);
2298  EVT InVT = InOp.getValueType();
2299  EVT VT = N->getValueType(0);
2300  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2301  SDLoc dl(N);
2302 
2303  switch (getTypeAction(InVT)) {
2305  break;
2307  // If the incoming type is a vector that is being promoted, then
2308  // we know that the elements are arranged differently and that we
2309  // must perform the conversion using a stack slot.
2310  if (InVT.isVector())
2311  break;
2312 
2313  // If the InOp is promoted to the same size, convert it. Otherwise,
2314  // fall out of the switch and widen the promoted input.
2315  InOp = GetPromotedInteger(InOp);
2316  InVT = InOp.getValueType();
2317  if (WidenVT.bitsEq(InVT))
2318  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2319  break;
2326  break;
2328  // If the InOp is widened to the same size, convert it. Otherwise, fall
2329  // out of the switch and widen the widened input.
2330  InOp = GetWidenedVector(InOp);
2331  InVT = InOp.getValueType();
2332  if (WidenVT.bitsEq(InVT))
2333  // The input widens to the same size. Convert to the widen value.
2334  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2335  break;
2336  }
2337 
2338  unsigned WidenSize = WidenVT.getSizeInBits();
2339  unsigned InSize = InVT.getSizeInBits();
2340  // x86mmx is not an acceptable vector element type, so don't try.
2341  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
2342  // Determine new input vector type. The new input vector type will use
2343  // the same element type (if its a vector) or use the input type as a
2344  // vector. It is the same size as the type to widen to.
2345  EVT NewInVT;
2346  unsigned NewNumElts = WidenSize / InSize;
2347  if (InVT.isVector()) {
2348  EVT InEltVT = InVT.getVectorElementType();
2349  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
2350  WidenSize / InEltVT.getSizeInBits());
2351  } else {
2352  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
2353  }
2354 
2355  if (TLI.isTypeLegal(NewInVT)) {
2356  // Because the result and the input are different vector types, widening
2357  // the result could create a legal type but widening the input might make
2358  // it an illegal type that might lead to repeatedly splitting the input
2359  // and then widening it. To avoid this, we widen the input only if
2360  // it results in a legal type.
2361  SmallVector<SDValue, 16> Ops(NewNumElts);
2362  SDValue UndefVal = DAG.getUNDEF(InVT);
2363  Ops[0] = InOp;
2364  for (unsigned i = 1; i < NewNumElts; ++i)
2365  Ops[i] = UndefVal;
2366 
2367  SDValue NewVec;
2368  if (InVT.isVector())
2369  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
2370  else
2371  NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops);
2372  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
2373  }
2374  }
2375 
2376  return CreateStackStoreLoad(InOp, WidenVT);
2377 }
2378 
2379 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
2380  SDLoc dl(N);
2381  // Build a vector with undefined for the new nodes.
2382  EVT VT = N->getValueType(0);
2383 
2384  // Integer BUILD_VECTOR operands may be larger than the node's vector element
2385  // type. The UNDEFs need to have the same type as the existing operands.
2386  EVT EltVT = N->getOperand(0).getValueType();
2387  unsigned NumElts = VT.getVectorNumElements();
2388 
2389  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2390  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2391 
2392  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
2393  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
2394  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
2395 
2396  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, NewOps);
2397 }
2398 
2399 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
2400  EVT InVT = N->getOperand(0).getValueType();
2401  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2402  SDLoc dl(N);
2403  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2404  unsigned NumInElts = InVT.getVectorNumElements();
2405  unsigned NumOperands = N->getNumOperands();
2406 
2407  bool InputWidened = false; // Indicates we need to widen the input.
2408  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
2409  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
2410  // Add undef vectors to widen to correct length.
2411  unsigned NumConcat = WidenVT.getVectorNumElements() /
2412  InVT.getVectorNumElements();
2413  SDValue UndefVal = DAG.getUNDEF(InVT);
2414  SmallVector<SDValue, 16> Ops(NumConcat);
2415  for (unsigned i=0; i < NumOperands; ++i)
2416  Ops[i] = N->getOperand(i);
2417  for (unsigned i = NumOperands; i != NumConcat; ++i)
2418  Ops[i] = UndefVal;
2419  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
2420  }
2421  } else {
2422  InputWidened = true;
2423  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
2424  // The inputs and the result are widen to the same value.
2425  unsigned i;
2426  for (i=1; i < NumOperands; ++i)
2427  if (N->getOperand(i).getOpcode() != ISD::UNDEF)
2428  break;
2429 
2430  if (i == NumOperands)
2431  // Everything but the first operand is an UNDEF so just return the
2432  // widened first operand.
2433  return GetWidenedVector(N->getOperand(0));
2434 
2435  if (NumOperands == 2) {
2436  // Replace concat of two operands with a shuffle.
2437  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
2438  for (unsigned i = 0; i < NumInElts; ++i) {
2439  MaskOps[i] = i;
2440  MaskOps[i + NumInElts] = i + WidenNumElts;
2441  }
2442  return DAG.getVectorShuffle(WidenVT, dl,
2443  GetWidenedVector(N->getOperand(0)),
2444  GetWidenedVector(N->getOperand(1)),
2445  &MaskOps[0]);
2446  }
2447  }
2448  }
2449 
2450  // Fall back to use extracts and build vector.
2451  EVT EltVT = WidenVT.getVectorElementType();
2452  SmallVector<SDValue, 16> Ops(WidenNumElts);
2453  unsigned Idx = 0;
2454  for (unsigned i=0; i < NumOperands; ++i) {
2455  SDValue InOp = N->getOperand(i);
2456  if (InputWidened)
2457  InOp = GetWidenedVector(InOp);
2458  for (unsigned j=0; j < NumInElts; ++j)
2459  Ops[Idx++] = DAG.getNode(
2460  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2461  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2462  }
2463  SDValue UndefVal = DAG.getUNDEF(EltVT);
2464  for (; Idx < WidenNumElts; ++Idx)
2465  Ops[Idx] = UndefVal;
2466  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops);
2467 }
2468 
2469 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
2470  SDLoc dl(N);
2471  SDValue InOp = N->getOperand(0);
2472  SDValue RndOp = N->getOperand(3);
2473  SDValue SatOp = N->getOperand(4);
2474 
2475  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2476  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2477 
2478  EVT InVT = InOp.getValueType();
2479  EVT InEltVT = InVT.getVectorElementType();
2480  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2481 
2482  SDValue DTyOp = DAG.getValueType(WidenVT);
2483  SDValue STyOp = DAG.getValueType(InWidenVT);
2484  ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
2485 
2486  unsigned InVTNumElts = InVT.getVectorNumElements();
2487  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2488  InOp = GetWidenedVector(InOp);
2489  InVT = InOp.getValueType();
2490  InVTNumElts = InVT.getVectorNumElements();
2491  if (InVTNumElts == WidenNumElts)
2492  return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
2493  SatOp, CvtCode);
2494  }
2495 
2496  if (TLI.isTypeLegal(InWidenVT)) {
2497  // Because the result and the input are different vector types, widening
2498  // the result could create a legal type but widening the input might make
2499  // it an illegal type that might lead to repeatedly splitting the input
2500  // and then widening it. To avoid this, we widen the input only if
2501  // it results in a legal type.
2502  if (WidenNumElts % InVTNumElts == 0) {
2503  // Widen the input and call convert on the widened input vector.
2504  unsigned NumConcat = WidenNumElts/InVTNumElts;
2505  SmallVector<SDValue, 16> Ops(NumConcat);
2506  Ops[0] = InOp;
2507  SDValue UndefVal = DAG.getUNDEF(InVT);
2508  for (unsigned i = 1; i != NumConcat; ++i)
2509  Ops[i] = UndefVal;
2510 
2511  InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, Ops);
2512  return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
2513  SatOp, CvtCode);
2514  }
2515 
2516  if (InVTNumElts % WidenNumElts == 0) {
2517  // Extract the input and convert the shorten input vector.
2518  InOp = DAG.getNode(
2519  ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp,
2520  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2521  return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
2522  SatOp, CvtCode);
2523  }
2524  }
2525 
2526  // Otherwise unroll into some nasty scalar code and rebuild the vector.
2527  SmallVector<SDValue, 16> Ops(WidenNumElts);
2528  EVT EltVT = WidenVT.getVectorElementType();
2529  DTyOp = DAG.getValueType(EltVT);
2530  STyOp = DAG.getValueType(InEltVT);
2531 
2532  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2533  unsigned i;
2534  for (i=0; i < MinElts; ++i) {
2535  SDValue ExtVal = DAG.getNode(
2536  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
2537  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2538  Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp,
2539  SatOp, CvtCode);
2540  }
2541 
2542  SDValue UndefVal = DAG.getUNDEF(EltVT);
2543  for (; i < WidenNumElts; ++i)
2544  Ops[i] = UndefVal;
2545 
2546  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops);
2547 }
2548 
2549 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
2550  EVT VT = N->getValueType(0);
2551  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2552  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2553  SDValue InOp = N->getOperand(0);
2554  SDValue Idx = N->getOperand(1);
2555  SDLoc dl(N);
2556 
2557  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2558  InOp = GetWidenedVector(InOp);
2559 
2560  EVT InVT = InOp.getValueType();
2561 
2562  // Check if we can just return the input vector after widening.
2563  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2564  if (IdxVal == 0 && InVT == WidenVT)
2565  return InOp;
2566 
2567  // Check if we can extract from the vector.
2568  unsigned InNumElts = InVT.getVectorNumElements();
2569  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2570  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
2571 
2572  // We could try widening the input to the right length but for now, extract
2573  // the original elements, fill the rest with undefs and build a vector.
2574  SmallVector<SDValue, 16> Ops(WidenNumElts);
2575  EVT EltVT = VT.getVectorElementType();
2576  unsigned NumElts = VT.getVectorNumElements();
2577  unsigned i;
2578  for (i=0; i < NumElts; ++i)
2579  Ops[i] =
2580  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2581  DAG.getConstant(IdxVal + i, dl,
2582  TLI.getVectorIdxTy(DAG.getDataLayout())));
2583 
2584  SDValue UndefVal = DAG.getUNDEF(EltVT);
2585  for (; i < WidenNumElts; ++i)
2586  Ops[i] = UndefVal;
2587  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops);
2588 }
2589 
2590 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
2591  SDValue InOp = GetWidenedVector(N->getOperand(0));
2592  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
2593  InOp.getValueType(), InOp,
2594  N->getOperand(1), N->getOperand(2));
2595 }
2596 
2597 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
2598  LoadSDNode *LD = cast<LoadSDNode>(N);
2599  ISD::LoadExtType ExtType = LD->getExtensionType();
2600 
2601  SDValue Result;
2602  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
2603  if (ExtType != ISD::NON_EXTLOAD)
2604  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2605  else
2606  Result = GenWidenVectorLoads(LdChain, LD);
2607 
2608  // If we generate a single load, we can use that for the chain. Otherwise,
2609  // build a factor node to remember the multiple loads are independent and
2610  // chain to that.
2611  SDValue NewChain;
2612  if (LdChain.size() == 1)
2613  NewChain = LdChain[0];
2614  else
2615  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
2616 
2617  // Modified the chain - switch anything that used the old chain to use
2618  // the new one.
2619  ReplaceValueWith(SDValue(N, 1), NewChain);
2620 
2621  return Result;
2622 }
2623 
2624 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
2625 
2626  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
2627  SDValue Mask = N->getMask();
2628  EVT MaskVT = Mask.getValueType();
2629  SDValue Src0 = GetWidenedVector(N->getSrc0());
2630  ISD::LoadExtType ExtType = N->getExtensionType();
2631  SDLoc dl(N);
2632 
2633  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
2634  Mask = GetWidenedVector(Mask);
2635  else {
2636  EVT BoolVT = getSetCCResultType(WidenVT);
2637 
2638  // We can't use ModifyToType() because we should fill the mask with
2639  // zeroes
2640  unsigned WidenNumElts = BoolVT.getVectorNumElements();
2641  unsigned MaskNumElts = MaskVT.getVectorNumElements();
2642 
2643  unsigned NumConcat = WidenNumElts / MaskNumElts;
2644  SmallVector<SDValue, 16> Ops(NumConcat);
2645  SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
2646  Ops[0] = Mask;
2647  for (unsigned i = 1; i != NumConcat; ++i)
2648  Ops[i] = ZeroVal;
2649 
2650  Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
2651  }
2652 
2653  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
2654  Mask, Src0, N->getMemoryVT(),
2655  N->getMemOperand(), ExtType);
2656  // Legalized the chain result - switch anything that used the old chain to
2657  // use the new one.
2658  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2659  return Res;
2660 }
2661 
2662 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
2663  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2664  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
2665  WidenVT, N->getOperand(0));
2666 }
2667 
2668 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
2669  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2670  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2671 
2672  SDValue Cond1 = N->getOperand(0);
2673  EVT CondVT = Cond1.getValueType();
2674  if (CondVT.isVector()) {
2675  EVT CondEltVT = CondVT.getVectorElementType();
2676  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
2677  CondEltVT, WidenNumElts);
2678  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
2679  Cond1 = GetWidenedVector(Cond1);
2680 
2681  // If we have to split the condition there is no point in widening the
2682  // select. This would result in an cycle of widening the select ->
2683  // widening the condition operand -> splitting the condition operand ->
2684  // splitting the select -> widening the select. Instead split this select
2685  // further and widen the resulting type.
2686  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
2687  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
2688  SDValue Res = ModifyToType(SplitSelect, WidenVT);
2689  return Res;
2690  }
2691 
2692  if (Cond1.getValueType() != CondWidenVT)
2693  Cond1 = ModifyToType(Cond1, CondWidenVT);
2694  }
2695 
2696  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
2697  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
2698  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
2699  return DAG.getNode(N->getOpcode(), SDLoc(N),
2700  WidenVT, Cond1, InOp1, InOp2);
2701 }
2702 
2703 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
2704  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
2705  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
2706  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2707  InOp1.getValueType(), N->getOperand(0),
2708  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
2709 }
2710 
2711 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
2712  assert(N->getValueType(0).isVector() ==
2713  N->getOperand(0).getValueType().isVector() &&
2714  "Scalar/Vector type mismatch");
2715  if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N);
2716 
2717  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2718  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2719  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2720  return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT,
2721  InOp1, InOp2, N->getOperand(2));
2722 }
2723 
2724 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
2725  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2726  return DAG.getUNDEF(WidenVT);
2727 }
2728 
2729 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
2730  EVT VT = N->getValueType(0);
2731  SDLoc dl(N);
2732 
2733  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2734  unsigned NumElts = VT.getVectorNumElements();
2735  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2736 
2737  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2738  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2739 
2740  // Adjust mask based on new input vector length.
2741  SmallVector<int, 16> NewMask;
2742  for (unsigned i = 0; i != NumElts; ++i) {
2743  int Idx = N->getMaskElt(i);
2744  if (Idx < (int)NumElts)
2745  NewMask.push_back(Idx);
2746  else
2747  NewMask.push_back(Idx - NumElts + WidenNumElts);
2748  }
2749  for (unsigned i = NumElts; i != WidenNumElts; ++i)
2750  NewMask.push_back(-1);
2751  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]);
2752 }
2753 
2754 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
2755  assert(N->getValueType(0).isVector() &&
2756  N->getOperand(0).getValueType().isVector() &&
2757  "Operands must be vectors");
2758  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2759  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2760 
2761  SDValue InOp1 = N->getOperand(0);
2762  EVT InVT = InOp1.getValueType();
2763  assert(InVT.isVector() && "can not widen non-vector type");
2764  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
2765  InVT.getVectorElementType(), WidenNumElts);
2766 
2767  // The input and output types often differ here, and it could be that while
2768  // we'd prefer to widen the result type, the input operands have been split.
2769  // In this case, we also need to split the result of this node as well.
2770  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
2771  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
2772  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
2773  return Res;
2774  }
2775 
2776  InOp1 = GetWidenedVector(InOp1);
2777  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2778 
2779  // Assume that the input and output will be widen appropriately. If not,
2780  // we will have to unroll it at some point.
2781  assert(InOp1.getValueType() == WidenInVT &&
2782  InOp2.getValueType() == WidenInVT &&
2783  "Input not widened to expected type!");
2784  (void)WidenInVT;
2785  return DAG.getNode(ISD::SETCC, SDLoc(N),
2786  WidenVT, InOp1, InOp2, N->getOperand(2));
2787 }
2788 
2789 
2790 //===----------------------------------------------------------------------===//
2791 // Widen Vector Operand
2792 //===----------------------------------------------------------------------===//
2793 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
2794  DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
2795  N->dump(&DAG);
2796  dbgs() << "\n");
2797  SDValue Res = SDValue();
2798 
2799  // See if the target wants to custom widen this node.
2800  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2801  return false;
2802 
2803  switch (N->getOpcode()) {
2804  default:
2805 #ifndef NDEBUG
2806  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
2807  N->dump(&DAG);
2808  dbgs() << "\n";
2809 #endif
2810  llvm_unreachable("Do not know how to widen this operator's operand!");
2811 
2812  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
2813  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
2814  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
2815  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
2816  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
2817  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
2818  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
2819 
2820  case ISD::ANY_EXTEND:
2821  case ISD::SIGN_EXTEND:
2822  case ISD::ZERO_EXTEND:
2823  Res = WidenVecOp_EXTEND(N);
2824  break;
2825 
2826  case ISD::FP_EXTEND:
2827  case ISD::FP_TO_SINT:
2828  case ISD::FP_TO_UINT:
2829  case ISD::SINT_TO_FP:
2830  case ISD::UINT_TO_FP:
2831  case ISD::TRUNCATE:
2832  Res = WidenVecOp_Convert(N);
2833  break;
2834  }
2835 
2836  // If Res is null, the sub-method took care of registering the result.
2837  if (!Res.getNode()) return false;
2838 
2839  // If the result is N, the sub-method updated N in place. Tell the legalizer
2840  // core about this.
2841  if (Res.getNode() == N)
2842  return true;
2843 
2844 
2845  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2846  "Invalid operand expansion");
2847 
2848  ReplaceValueWith(SDValue(N, 0), Res);
2849  return false;
2850 }
2851 
2852 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
2853  SDLoc DL(N);
2854  EVT VT = N->getValueType(0);
2855 
2856  SDValue InOp = N->getOperand(0);
2857  // If some legalization strategy other than widening is used on the operand,
2858  // we can't safely assume that just extending the low lanes is the correct
2859  // transformation.
2860  if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector)
2861  return WidenVecOp_Convert(N);
2862  InOp = GetWidenedVector(InOp);
2863  assert(VT.getVectorNumElements() <
2865  "Input wasn't widened!");
2866 
2867  // We may need to further widen the operand until it has the same total
2868  // vector size as the result.
2869  EVT InVT = InOp.getValueType();
2870  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
2871  EVT InEltVT = InVT.getVectorElementType();
2872  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
2873  EVT FixedVT = (MVT::SimpleValueType)i;
2874  EVT FixedEltVT = FixedVT.getVectorElementType();
2875  if (TLI.isTypeLegal(FixedVT) &&
2876  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
2877  FixedEltVT == InEltVT) {
2878  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
2879  "Not enough elements in the fixed type for the operand!");
2880  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
2881  "We can't have the same type as we started with!");
2882  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
2883  InOp = DAG.getNode(
2884  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
2885  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2886  else
2887  InOp = DAG.getNode(
2888  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
2889  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2890  break;
2891  }
2892  }
2893  InVT = InOp.getValueType();
2894  if (InVT.getSizeInBits() != VT.getSizeInBits())
2895  // We couldn't find a legal vector type that was a widening of the input
2896  // and could be extended in-register to the result type, so we have to
2897  // scalarize.
2898  return WidenVecOp_Convert(N);
2899  }
2900 
2901  // Use special DAG nodes to represent the operation of extending the
2902  // low lanes.
2903  switch (N->getOpcode()) {
2904  default:
2905  llvm_unreachable("Extend legalization on on extend operation!");
2906  case ISD::ANY_EXTEND:
2907  return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
2908  case ISD::SIGN_EXTEND:
2909  return DAG.getSignExtendVectorInReg(InOp, DL, VT);
2910  case ISD::ZERO_EXTEND:
2911  return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
2912  }
2913 }
2914 
2915 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
2916  // Since the result is legal and the input is illegal, it is unlikely
2917  // that we can fix the input to a legal type so unroll the convert
2918  // into some scalar code and create a nasty build vector.
2919  EVT VT = N->getValueType(0);
2920  EVT EltVT = VT.getVectorElementType();
2921  SDLoc dl(N);
2922  unsigned NumElts = VT.getVectorNumElements();
2923  SDValue InOp = N->getOperand(0);
2924  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2925  InOp = GetWidenedVector(InOp);
2926  EVT InVT = InOp.getValueType();
2927  EVT InEltVT = InVT.getVectorElementType();
2928 
2929  unsigned Opcode = N->getOpcode();
2930  SmallVector<SDValue, 16> Ops(NumElts);
2931  for (unsigned i=0; i < NumElts; ++i)
2932  Ops[i] = DAG.getNode(
2933  Opcode, dl, EltVT,
2934  DAG.getNode(
2935  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
2936  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2937 
2938  return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
2939 }
2940 
2941 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
2942  EVT VT = N->getValueType(0);
2943  SDValue InOp = GetWidenedVector(N->getOperand(0));
2944  EVT InWidenVT = InOp.getValueType();
2945  SDLoc dl(N);
2946 
2947  // Check if we can convert between two legal vector types and extract.
2948  unsigned InWidenSize = InWidenVT.getSizeInBits();
2949  unsigned Size = VT.getSizeInBits();
2950  // x86mmx is not an acceptable vector element type, so don't try.
2951  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
2952  unsigned NewNumElts = InWidenSize / Size;
2953  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
2954  if (TLI.isTypeLegal(NewVT)) {
2955  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
2956  return DAG.getNode(
2957  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
2958  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2959  }
2960  }
2961 
2962  return CreateStackStoreLoad(InOp, VT);
2963 }
2964 
2965 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
2966  // If the input vector is not legal, it is likely that we will not find a
2967  // legal vector of the same size. Replace the concatenate vector with a
2968  // nasty build vector.
2969  EVT VT = N->getValueType(0);
2970  EVT EltVT = VT.getVectorElementType();
2971  SDLoc dl(N);
2972  unsigned NumElts = VT.getVectorNumElements();
2973  SmallVector<SDValue, 16> Ops(NumElts);
2974 
2975  EVT InVT = N->getOperand(0).getValueType();
2976  unsigned NumInElts = InVT.getVectorNumElements();
2977 
2978  unsigned Idx = 0;
2979  unsigned NumOperands = N->getNumOperands();
2980  for (unsigned i=0; i < NumOperands; ++i) {
2981  SDValue InOp = N->getOperand(i);
2982  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2983  InOp = GetWidenedVector(InOp);
2984  for (unsigned j=0; j < NumInElts; ++j)
2985  Ops[Idx++] = DAG.getNode(
2986  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2987  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2988  }
2989  return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
2990 }
2991 
2992 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2993  SDValue InOp = GetWidenedVector(N->getOperand(0));
2994  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
2995  N->getValueType(0), InOp, N->getOperand(1));
2996 }
2997 
2998 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2999  SDValue InOp = GetWidenedVector(N->getOperand(0));
3000  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3001  N->getValueType(0), InOp, N->getOperand(1));
3002 }
3003 
3004 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
3005  // We have to widen the value but we want only to store the original
3006  // vector type.
3007  StoreSDNode *ST = cast<StoreSDNode>(N);
3008 
3009  SmallVector<SDValue, 16> StChain;
3010  if (ST->isTruncatingStore())
3011  GenWidenVectorTruncStores(StChain, ST);
3012  else
3013  GenWidenVectorStores(StChain, ST);
3014 
3015  if (StChain.size() == 1)
3016  return StChain[0];
3017  else
3018  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
3019 }
3020 
3021 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
3022  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
3023  SDValue Mask = MST->getMask();
3024  EVT MaskVT = Mask.getValueType();
3025  SDValue StVal = MST->getValue();
3026  // Widen the value
3027  SDValue WideVal = GetWidenedVector(StVal);
3028  SDLoc dl(N);
3029 
3030  if (OpNo == 2 || getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3031  Mask = GetWidenedVector(Mask);
3032  else {
3033  // The mask should be widened as well
3034  EVT BoolVT = getSetCCResultType(WideVal.getValueType());
3035  // We can't use ModifyToType() because we should fill the mask with
3036  // zeroes
3037  unsigned WidenNumElts = BoolVT.getVectorNumElements();
3038  unsigned MaskNumElts = MaskVT.getVectorNumElements();
3039 
3040  unsigned NumConcat = WidenNumElts / MaskNumElts;
3041  SmallVector<SDValue, 16> Ops(NumConcat);
3042  SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
3043  Ops[0] = Mask;
3044  for (unsigned i = 1; i != NumConcat; ++i)
3045  Ops[i] = ZeroVal;
3046 
3047  Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
3048  }
3049  assert(Mask.getValueType().getVectorNumElements() ==
3050  WideVal.getValueType().getVectorNumElements() &&
3051  "Mask and data vectors should have the same number of elements");
3052  return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
3053  Mask, MST->getMemoryVT(), MST->getMemOperand(),
3054  false);
3055 }
3056 
3057 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
3058  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
3059  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3060  SDLoc dl(N);
3061 
3062  // WARNING: In this code we widen the compare instruction with garbage.
3063  // This garbage may contain denormal floats which may be slow. Is this a real
3064  // concern ? Should we zero the unused lanes if this is a float compare ?
3065 
3066  // Get a new SETCC node to compare the newly widened operands.
3067  // Only some of the compared elements are legal.
3068  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3069  InOp0.getValueType());
3070  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
3071  SVT, InOp0, InOp1, N->getOperand(2));
3072 
3073  // Extract the needed results from the result vector.
3074  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
3075  SVT.getVectorElementType(),
3077  SDValue CC = DAG.getNode(
3078  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
3079  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3080 
3081  return PromoteTargetBoolean(CC, N->getValueType(0));
3082 }
3083 
3084 
3085 //===----------------------------------------------------------------------===//
3086 // Vector Widening Utilities
3087 //===----------------------------------------------------------------------===//
3088 
3089 // Utility function to find the type to chop up a widen vector for load/store
3090 // TLI: Target lowering used to determine legal types.
3091 // Width: Width left need to load/store.
3092 // WidenVT: The widen vector type to load to/store from
3093 // Align: If 0, don't allow use of a wider type
3094 // WidenEx: If Align is not 0, the amount additional we can load/store from.
3095 
3096 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
3097  unsigned Width, EVT WidenVT,
3098  unsigned Align = 0, unsigned WidenEx = 0) {
3099  EVT WidenEltVT = WidenVT.getVectorElementType();
3100  unsigned WidenWidth = WidenVT.getSizeInBits();
3101  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
3102  unsigned AlignInBits = Align*8;
3103 
3104  // If we have one element to load/store, return it.
3105  EVT RetVT = WidenEltVT;
3106  if (Width == WidenEltWidth)
3107  return RetVT;
3108 
3109  // See if there is larger legal integer than the element type to load/store
3110  unsigned VT;
3111  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
3112  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
3113  EVT MemVT((MVT::SimpleValueType) VT);
3114  unsigned MemVTWidth = MemVT.getSizeInBits();
3115  if (MemVT.getSizeInBits() <= WidenEltWidth)
3116  break;
3117  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
3118  if ((Action == TargetLowering::TypeLegal ||
3120  (WidenWidth % MemVTWidth) == 0 &&
3121  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3122  (MemVTWidth <= Width ||
3123  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3124  RetVT = MemVT;
3125  break;
3126  }
3127  }
3128 
3129  // See if there is a larger vector type to load/store that has the same vector
3130  // element type and is evenly divisible with the WidenVT.
3131  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
3132  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
3133  EVT MemVT = (MVT::SimpleValueType) VT;
3134  unsigned MemVTWidth = MemVT.getSizeInBits();
3135  if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
3136  (WidenWidth % MemVTWidth) == 0 &&
3137  isPowerOf2_32(WidenWidth / MemVTWidth) &&
3138  (MemVTWidth <= Width ||
3139  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3140  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
3141  return MemVT;
3142  }
3143  }
3144 
3145  return RetVT;
3146 }
3147 
3148 // Builds a vector type from scalar loads
3149 // VecTy: Resulting Vector type
3150 // LDOps: Load operators to build a vector type
3151 // [Start,End) the list of loads to use.
3153  SmallVectorImpl<SDValue> &LdOps,
3154  unsigned Start, unsigned End) {
3155  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3156  SDLoc dl(LdOps[Start]);
3157  EVT LdTy = LdOps[Start].getValueType();
3158  unsigned Width = VecTy.getSizeInBits();
3159  unsigned NumElts = Width / LdTy.getSizeInBits();
3160  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
3161 
3162  unsigned Idx = 1;
3163  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
3164 
3165  for (unsigned i = Start + 1; i != End; ++i) {
3166  EVT NewLdTy = LdOps[i].getValueType();
3167  if (NewLdTy != LdTy) {
3168  NumElts = Width / NewLdTy.getSizeInBits();
3169  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
3170  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
3171  // Readjust position and vector position based on new load type
3172  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
3173  LdTy = NewLdTy;
3174  }
3175  VecOp = DAG.getNode(
3176  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
3177  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3178  }
3179  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
3180 }
3181 
3182 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
3183  LoadSDNode *LD) {
3184  // The strategy assumes that we can efficiently load powers of two widths.
3185  // The routines chops the vector into the largest vector loads with the same
3186  // element type or scalar loads and then recombines it to the widen vector
3187  // type.
3188  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3189  unsigned WidenWidth = WidenVT.getSizeInBits();
3190  EVT LdVT = LD->getMemoryVT();
3191  SDLoc dl(LD);
3192  assert(LdVT.isVector() && WidenVT.isVector());
3193  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
3194 
3195  // Load information
3196  SDValue Chain = LD->getChain();
3197  SDValue BasePtr = LD->getBasePtr();
3198  unsigned Align = LD->getAlignment();
3199  bool isVolatile = LD->isVolatile();
3200  bool isNonTemporal = LD->isNonTemporal();
3201  bool isInvariant = LD->isInvariant();
3202  AAMDNodes AAInfo = LD->getAAInfo();
3203 
3204  int LdWidth = LdVT.getSizeInBits();
3205  int WidthDiff = WidenWidth - LdWidth; // Difference
3206  unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads
3207 
3208  // Find the vector type that can load from.
3209  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3210  int NewVTWidth = NewVT.getSizeInBits();
3211  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
3212  isVolatile, isNonTemporal, isInvariant, Align,
3213  AAInfo);
3214  LdChain.push_back(LdOp.getValue(1));
3215 
3216  // Check if we can load the element with one instruction
3217  if (LdWidth <= NewVTWidth) {
3218  if (!NewVT.isVector()) {
3219  unsigned NumElts = WidenWidth / NewVTWidth;
3220  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
3221  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
3222  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
3223  }
3224  if (NewVT == WidenVT)
3225  return LdOp;
3226 
3227  assert(WidenWidth % NewVTWidth == 0);
3228  unsigned NumConcat = WidenWidth / NewVTWidth;
3229  SmallVector<SDValue, 16> ConcatOps(NumConcat);
3230  SDValue UndefVal = DAG.getUNDEF(NewVT);
3231  ConcatOps[0] = LdOp;
3232  for (unsigned i = 1; i != NumConcat; ++i)
3233  ConcatOps[i] = UndefVal;
3234  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
3235  }
3236 
3237  // Load vector by using multiple loads from largest vector to scalar
3239  LdOps.push_back(LdOp);
3240 
3241  LdWidth -= NewVTWidth;
3242  unsigned Offset = 0;
3243 
3244  while (LdWidth > 0) {
3245  unsigned Increment = NewVTWidth / 8;
3246  Offset += Increment;
3247  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3248  DAG.getConstant(Increment, dl, BasePtr.getValueType()));
3249 
3250  SDValue L;
3251  if (LdWidth < NewVTWidth) {
3252  // Our current type we are using is too large, find a better size
3253  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3254  NewVTWidth = NewVT.getSizeInBits();
3255  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3256  LD->getPointerInfo().getWithOffset(Offset), isVolatile,
3257  isNonTemporal, isInvariant, MinAlign(Align, Increment),
3258  AAInfo);
3259  LdChain.push_back(L.getValue(1));
3260  if (L->getValueType(0).isVector()) {
3262  Loads.push_back(L);
3263  unsigned size = L->getValueSizeInBits(0);
3264  while (size < LdOp->getValueSizeInBits(0)) {
3265  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
3266  size += L->getValueSizeInBits(0);
3267  }
3268  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
3269  }
3270  } else {
3271  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
3272  LD->getPointerInfo().getWithOffset(Offset), isVolatile,
3273  isNonTemporal, isInvariant, MinAlign(Align, Increment),
3274  AAInfo);
3275  LdChain.push_back(L.getValue(1));
3276  }
3277 
3278  LdOps.push_back(L);
3279 
3280 
3281  LdWidth -= NewVTWidth;
3282  }
3283 
3284  // Build the vector from the loads operations
3285  unsigned End = LdOps.size();
3286  if (!LdOps[0].getValueType().isVector())
3287  // All the loads are scalar loads.
3288  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
3289 
3290  // If the load contains vectors, build the vector using concat vector.
3291  // All of the vectors used to loads are power of 2 and the scalars load
3292  // can be combined to make a power of 2 vector.
3293  SmallVector<SDValue, 16> ConcatOps(End);
3294  int i = End - 1;
3295  int Idx = End;
3296  EVT LdTy = LdOps[i].getValueType();
3297  // First combine the scalar loads to a vector
3298  if (!LdTy.isVector()) {
3299  for (--i; i >= 0; --i) {
3300  LdTy = LdOps[i].getValueType();
3301  if (LdTy.isVector())
3302  break;
3303  }
3304  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End);
3305  }
3306  ConcatOps[--Idx] = LdOps[i];
3307  for (--i; i >= 0; --i) {
3308  EVT NewLdTy = LdOps[i].getValueType();
3309  if (NewLdTy != LdTy) {
3310  // Create a larger vector
3311  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
3312  makeArrayRef(&ConcatOps[Idx], End - Idx));
3313  Idx = End - 1;
3314  LdTy = NewLdTy;
3315  }
3316  ConcatOps[--Idx] = LdOps[i];
3317  }
3318 
3319  if (WidenWidth == LdTy.getSizeInBits()*(End - Idx))
3320  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3321  makeArrayRef(&ConcatOps[Idx], End - Idx));
3322 
3323  // We need to fill the rest with undefs to build the vector
3324  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
3325  SmallVector<SDValue, 16> WidenOps(NumOps);
3326  SDValue UndefVal = DAG.getUNDEF(LdTy);
3327  {
3328  unsigned i = 0;
3329  for (; i != End-Idx; ++i)
3330  WidenOps[i] = ConcatOps[Idx+i];
3331  for (; i != NumOps; ++i)
3332  WidenOps[i] = UndefVal;
3333  }
3334  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
3335 }
3336 
3337 SDValue
3338 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
3339  LoadSDNode *LD,
3340  ISD::LoadExtType ExtType) {
3341  // For extension loads, it may not be more efficient to chop up the vector
3342  // and then extended it. Instead, we unroll the load and build a new vector.
3343  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
3344  EVT LdVT = LD->getMemoryVT();
3345  SDLoc dl(LD);
3346  assert(LdVT.isVector() && WidenVT.isVector());
3347 
3348  // Load information
3349  SDValue Chain = LD->getChain();
3350  SDValue BasePtr = LD->getBasePtr();
3351  unsigned Align = LD->getAlignment();
3352  bool isVolatile = LD->isVolatile();
3353  bool isNonTemporal = LD->isNonTemporal();
3354  bool isInvariant = LD->isInvariant();
3355  AAMDNodes AAInfo = LD->getAAInfo();
3356 
3357  EVT EltVT = WidenVT.getVectorElementType();
3358  EVT LdEltVT = LdVT.getVectorElementType();
3359  unsigned NumElts = LdVT.getVectorNumElements();
3360 
3361  // Load each element and widen
3362  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3363  SmallVector<SDValue, 16> Ops(WidenNumElts);
3364  unsigned Increment = LdEltVT.getSizeInBits() / 8;
3365  Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr,
3366  LD->getPointerInfo(),
3367  LdEltVT, isVolatile, isNonTemporal, isInvariant,
3368  Align, AAInfo);
3369  LdChain.push_back(Ops[0].getValue(1));
3370  unsigned i = 0, Offset = Increment;
3371  for (i=1; i < NumElts; ++i, Offset += Increment) {
3372  SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
3373  BasePtr,
3374  DAG.getConstant(Offset, dl,
3375  BasePtr.getValueType()));
3376  Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
3377  LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
3378  isVolatile, isNonTemporal, isInvariant, Align,
3379  AAInfo);
3380  LdChain.push_back(Ops[i].getValue(1));
3381  }
3382 
3383  // Fill the rest with undefs
3384  SDValue UndefVal = DAG.getUNDEF(EltVT);
3385  for (; i != WidenNumElts; ++i)
3386  Ops[i] = UndefVal;
3387 
3388  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops);
3389 }
3390 
3391 
3392 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
3393  StoreSDNode *ST) {
3394  // The strategy assumes that we can efficiently store powers of two widths.
3395  // The routines chops the vector into the largest vector stores with the same
3396  // element type or scalar stores.
3397  SDValue Chain = ST->getChain();
3398  SDValue BasePtr = ST->getBasePtr();
3399  unsigned Align = ST->getAlignment();
3400  bool isVolatile = ST->isVolatile();
3401  bool isNonTemporal = ST->isNonTemporal();
3402  AAMDNodes AAInfo = ST->getAAInfo();
3403  SDValue ValOp = GetWidenedVector(ST->getValue());
3404  SDLoc dl(ST);
3405 
3406  EVT StVT = ST->getMemoryVT();
3407  unsigned StWidth = StVT.getSizeInBits();
3408  EVT ValVT = ValOp.getValueType();
3409  unsigned ValWidth = ValVT.getSizeInBits();
3410  EVT ValEltVT = ValVT.getVectorElementType();
3411  unsigned ValEltWidth = ValEltVT.getSizeInBits();
3412  assert(StVT.getVectorElementType() == ValEltVT);
3413 
3414  int Idx = 0; // current index to store
3415  unsigned Offset = 0; // offset from base to store
3416  while (StWidth != 0) {
3417  // Find the largest vector type we can store with
3418  EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
3419  unsigned NewVTWidth = NewVT.getSizeInBits();
3420  unsigned Increment = NewVTWidth / 8;
3421  if (NewVT.isVector()) {
3422  unsigned NumVTElts = NewVT.getVectorNumElements();
3423  do {
3424  SDValue EOp = DAG.getNode(
3425  ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
3426  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3427  StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr,
3428  ST->getPointerInfo().getWithOffset(Offset),
3429  isVolatile, isNonTemporal,
3430  MinAlign(Align, Offset), AAInfo));
3431  StWidth -= NewVTWidth;
3432  Offset += Increment;
3433  Idx += NumVTElts;
3434  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3435  DAG.getConstant(Increment, dl,
3436  BasePtr.getValueType()));
3437  } while (StWidth != 0 && StWidth >= NewVTWidth);
3438  } else {
3439  // Cast the vector to the scalar type we can store
3440  unsigned NumElts = ValWidth / NewVTWidth;
3441  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
3442  SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
3443  // Readjust index position based on new vector type
3444  Idx = Idx * ValEltWidth / NewVTWidth;
3445  do {
3446  SDValue EOp = DAG.getNode(
3447  ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
3448  DAG.getConstant(Idx++, dl,
3449  TLI.getVectorIdxTy(DAG.getDataLayout())));
3450  StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr,
3451  ST->getPointerInfo().getWithOffset(Offset),
3452  isVolatile, isNonTemporal,
3453  MinAlign(Align, Offset), AAInfo));
3454  StWidth -= NewVTWidth;
3455  Offset += Increment;
3456  BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
3457  DAG.getConstant(Increment, dl,
3458  BasePtr.getValueType()));
3459  } while (StWidth != 0 && StWidth >= NewVTWidth);
3460  // Restore index back to be relative to the original widen element type
3461  Idx = Idx * NewVTWidth / ValEltWidth;
3462  }
3463  }
3464 }
3465 
3466 void
3467 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
3468  StoreSDNode *ST) {
3469  // For extension loads, it may not be more efficient to truncate the vector
3470  // and then store it. Instead, we extract each element and then store it.
3471  SDValue Chain = ST->getChain();
3472  SDValue BasePtr = ST->getBasePtr();
3473  unsigned Align = ST->getAlignment();
3474  bool isVolatile = ST->isVolatile();
3475  bool isNonTemporal = ST->isNonTemporal();
3476  AAMDNodes AAInfo = ST->getAAInfo();
3477  SDValue ValOp = GetWidenedVector(ST->getValue());
3478  SDLoc dl(ST);
3479 
3480  EVT StVT = ST->getMemoryVT();
3481  EVT ValVT = ValOp.getValueType();
3482 
3483  // It must be true that we the widen vector type is bigger than where
3484  // we need to store.
3485  assert(StVT.isVector() && ValOp.getValueType().isVector());
3486  assert(StVT.bitsLT(ValOp.getValueType()));
3487 
3488  // For truncating stores, we can not play the tricks of chopping legal
3489  // vector types and bit cast it to the right type. Instead, we unroll
3490  // the store.
3491  EVT StEltVT = StVT.getVectorElementType();
3492  EVT ValEltVT = ValVT.getVectorElementType();
3493  unsigned Increment = ValEltVT.getSizeInBits() / 8;
3494  unsigned NumElts = StVT.getVectorNumElements();
3495  SDValue EOp = DAG.getNode(
3496  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
3497  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3498  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
3499  ST->getPointerInfo(), StEltVT,
3500  isVolatile, isNonTemporal, Align,
3501  AAInfo));
3502  unsigned Offset = Increment;
3503  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
3504  SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
3505  BasePtr,
3506  DAG.getConstant(Offset, dl,
3507  BasePtr.getValueType()));
3508  SDValue EOp = DAG.getNode(
3509  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
3510  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3511  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr,
3512  ST->getPointerInfo().getWithOffset(Offset),
3513  StEltVT, isVolatile, isNonTemporal,
3514  MinAlign(Align, Offset), AAInfo));
3515  }
3516 }
3517 
3518 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
3519 /// input vector must have the same element type as NVT.
3520 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) {
3521  // Note that InOp might have been widened so it might already have
3522  // the right width or it might need be narrowed.
3523  EVT InVT = InOp.getValueType();
3524  assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
3525  "input and widen element type must match");
3526  SDLoc dl(InOp);
3527 
3528  // Check if InOp already has the right width.
3529  if (InVT == NVT)
3530  return InOp;
3531 
3532  unsigned InNumElts = InVT.getVectorNumElements();
3533  unsigned WidenNumElts = NVT.getVectorNumElements();
3534  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
3535  unsigned NumConcat = WidenNumElts / InNumElts;
3536  SmallVector<SDValue, 16> Ops(NumConcat);
3537  SDValue UndefVal = DAG.getUNDEF(InVT);
3538  Ops[0] = InOp;
3539  for (unsigned i = 1; i != NumConcat; ++i)
3540  Ops[i] = UndefVal;
3541 
3542  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
3543  }
3544 
3545  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
3546  return DAG.getNode(
3547  ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
3548  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3549 
3550  // Fall back to extract and build.
3551  SmallVector<SDValue, 16> Ops(WidenNumElts);
3552  EVT EltVT = NVT.getVectorElementType();
3553  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
3554  unsigned Idx;
3555  for (Idx = 0; Idx < MinNumElts; ++Idx)
3556  Ops[Idx] = DAG.getNode(
3557  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3558  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3559 
3560  SDValue UndefVal = DAG.getUNDEF(EltVT);
3561  for ( ; Idx < WidenNumElts; ++Idx)
3562  Ops[Idx] = UndefVal;
3563  return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Ops);
3564 }
SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, bool isNonTemporal, bool isVolatile, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:477
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:450
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
The memory access reads data.
SDValue getValue(unsigned R) const
const SDValue & getValue() const
The memory access writes data.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
LLVMContext * getContext() const
Definition: SelectionDAG.h:289
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
void dump() const
Dump this node, for debugging.
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:292
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:301
SDValue getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:684
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getSrc0() const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:181
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
SDValue getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:318
CvtCode
CvtCode enum - This enum defines the various converts CONVERT_RNDSAT supports.
Definition: ISDOpcodes.h:885
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:287
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:357
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:189
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:115
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
MachineMemOperand - A description of a memory reference used in the backend.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:308
Shift and rotation operations.
Definition: ISDOpcodes.h:332
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:283
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:210
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
int getMaskElt(unsigned Idx) const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
ISD::LoadExtType getExtensionType() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:110
This class is used to represent an MSTORE node.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:216
load Combine Adjacent Loads
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:393
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:351
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:191
SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, const int *MaskElts)
Return an ISD::VECTOR_SHUFFLE node.
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:659
EVT getMemoryVT() const
Return the type of the in-memory value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:284
UNDEF - An undefined node.
Definition: ISDOpcodes.h:169
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:436
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:262
LLVM_CONSTEXPR size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:247
SDNode * getNode() const
get the SDNode which holds the desired result
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:245
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:41
Simple binary floating point operators.
Definition: ISDOpcodes.h:237
bool isNonTemporal() const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:780
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:267
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
static EVT getFloatingPointVT(unsigned BitWidth)
getFloatingPointVT - Returns the EVT that represents a floating point type with the given number of b...
Definition: ValueTypes.h:55
SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:436
unsigned getOpcode() const
const SDValue & getBasePtr() const
bool isVolatile() const
const SDValue & getValue() const
SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:338
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:468
EVT - Extended Value Type.
Definition: ValueTypes.h:31
const SDValue & getMask() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:171
SDValue getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType)
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
MachinePointerInfo - This class contains a discriminated union of information about pointers in memor...
SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTrunc)
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const MachinePointerInfo & getPointerInfo() const
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
SDValue getConvertRndSat(EVT VT, SDLoc dl, SDValue Val, SDValue DTy, SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code)
Returns the ConvertRndSat Note: Avoid using this node because it may disappear in the future and most...
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:273
bool isInvariant() const
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:179
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:465
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:548
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:335
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
const SDValue & getValue() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:342
op_iterator op_begin() const
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:383
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:386
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:250
iterator_range< value_op_iterator > op_values() const
const SDValue & getMask() const
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:552
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:321
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:401
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:518
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
#define N
op_iterator op_end() const
void size_t size
This class is used to represent an MSCATTER node.
const SDValue & getIndex() const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
CONVERT_RNDSAT - This operator is used to support various conversions between various types (float...
Definition: ISDOpcodes.h:494
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:279
SDValue getSelect(SDLoc DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:739
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:105
This class is used to represent an MLOAD node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:240
const SDValue & getBasePtr() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:287
#define DEBUG(X)
Definition: Debug.h:92
SDValue getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:354
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:365
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:188
static bool isVolatile(Instruction *Inst)
Conversion operators.
Definition: ISDOpcodes.h:380
BooleanContent
Enum that describes how the target represents true/false values.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:338
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:389
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool isBigEndian() const
Definition: DataLayout.h:218
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:506
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:309
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:314
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:225
This class is used to represent ISD::LOAD nodes.