LLVM  8.0.0svn
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16 // types.
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "legalize-types"
30 
31 //===----------------------------------------------------------------------===//
32 // Result Vector Scalarization: <1 x ty> -> ty.
33 //===----------------------------------------------------------------------===//
34 
35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
37  dbgs() << "\n");
38  SDValue R = SDValue();
39 
40  switch (N->getOpcode()) {
41  default:
42 #ifndef NDEBUG
43  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
44  N->dump(&DAG);
45  dbgs() << "\n";
46 #endif
47  report_fatal_error("Do not know how to scalarize the result of this "
48  "operator!\n");
49 
50  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
51  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
52  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
53  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
54  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
55  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
56  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
70  R = ScalarizeVecRes_VecInregOp(N);
71  break;
72  case ISD::ANY_EXTEND:
73  case ISD::BITREVERSE:
74  case ISD::BSWAP:
75  case ISD::CTLZ:
77  case ISD::CTPOP:
78  case ISD::CTTZ:
80  case ISD::FABS:
81  case ISD::FCEIL:
82  case ISD::FCOS:
83  case ISD::FEXP:
84  case ISD::FEXP2:
85  case ISD::FFLOOR:
86  case ISD::FLOG:
87  case ISD::FLOG10:
88  case ISD::FLOG2:
89  case ISD::FNEARBYINT:
90  case ISD::FNEG:
91  case ISD::FP_EXTEND:
92  case ISD::FP_TO_SINT:
93  case ISD::FP_TO_UINT:
94  case ISD::FRINT:
95  case ISD::FROUND:
96  case ISD::FSIN:
97  case ISD::FSQRT:
98  case ISD::FTRUNC:
99  case ISD::SIGN_EXTEND:
100  case ISD::SINT_TO_FP:
101  case ISD::TRUNCATE:
102  case ISD::UINT_TO_FP:
103  case ISD::ZERO_EXTEND:
104  case ISD::FCANONICALIZE:
105  R = ScalarizeVecRes_UnaryOp(N);
106  break;
107 
108  case ISD::ADD:
109  case ISD::AND:
110  case ISD::FADD:
111  case ISD::FCOPYSIGN:
112  case ISD::FDIV:
113  case ISD::FMUL:
114  case ISD::FMINNUM:
115  case ISD::FMAXNUM:
116  case ISD::FMINNUM_IEEE:
117  case ISD::FMAXNUM_IEEE:
118  case ISD::FMINNAN:
119  case ISD::FMAXNAN:
120  case ISD::SMIN:
121  case ISD::SMAX:
122  case ISD::UMIN:
123  case ISD::UMAX:
124 
125  case ISD::SADDSAT:
126  case ISD::UADDSAT:
127 
128  case ISD::FPOW:
129  case ISD::FREM:
130  case ISD::FSUB:
131  case ISD::MUL:
132  case ISD::OR:
133  case ISD::SDIV:
134  case ISD::SREM:
135  case ISD::SUB:
136  case ISD::UDIV:
137  case ISD::UREM:
138  case ISD::XOR:
139  case ISD::SHL:
140  case ISD::SRA:
141  case ISD::SRL:
142  R = ScalarizeVecRes_BinOp(N);
143  break;
144  case ISD::FMA:
145  R = ScalarizeVecRes_TernaryOp(N);
146  break;
147  case ISD::STRICT_FADD:
148  case ISD::STRICT_FSUB:
149  case ISD::STRICT_FMUL:
150  case ISD::STRICT_FDIV:
151  case ISD::STRICT_FREM:
152  case ISD::STRICT_FSQRT:
153  case ISD::STRICT_FMA:
154  case ISD::STRICT_FPOW:
155  case ISD::STRICT_FPOWI:
156  case ISD::STRICT_FSIN:
157  case ISD::STRICT_FCOS:
158  case ISD::STRICT_FEXP:
159  case ISD::STRICT_FEXP2:
160  case ISD::STRICT_FLOG:
161  case ISD::STRICT_FLOG10:
162  case ISD::STRICT_FLOG2:
163  case ISD::STRICT_FRINT:
165  R = ScalarizeVecRes_StrictFPOp(N);
166  break;
167  }
168 
169  // If R is null, the sub-method took care of registering the result.
170  if (R.getNode())
171  SetScalarizedVector(SDValue(N, ResNo), R);
172 }
173 
174 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
175  SDValue LHS = GetScalarizedVector(N->getOperand(0));
176  SDValue RHS = GetScalarizedVector(N->getOperand(1));
177  return DAG.getNode(N->getOpcode(), SDLoc(N),
178  LHS.getValueType(), LHS, RHS, N->getFlags());
179 }
180 
181 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
182  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
183  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
184  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
185  return DAG.getNode(N->getOpcode(), SDLoc(N),
186  Op0.getValueType(), Op0, Op1, Op2);
187 }
188 
189 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
191  unsigned NumOpers = N->getNumOperands();
192  SDValue Chain = N->getOperand(0);
193  EVT ValueVTs[] = {VT, MVT::Other};
194  SDLoc dl(N);
195 
197 
198  // The Chain is the first operand.
199  Opers.push_back(Chain);
200 
201  // Now process the remaining operands.
202  for (unsigned i = 1; i < NumOpers; ++i) {
203  SDValue Oper = N->getOperand(i);
204 
205  if (Oper.getValueType().isVector())
206  Oper = GetScalarizedVector(Oper);
207 
208  Opers.push_back(Oper);
209  }
210 
211  SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
212 
213  // Legalize the chain result - switch anything that used the old chain to
214  // use the new one.
215  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
216  return Result;
217 }
218 
219 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
220  unsigned ResNo) {
221  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
222  return GetScalarizedVector(Op);
223 }
224 
225 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
226  SDValue Op = N->getOperand(0);
227  if (Op.getValueType().isVector()
228  && Op.getValueType().getVectorNumElements() == 1
229  && !isSimpleLegalType(Op.getValueType()))
230  Op = GetScalarizedVector(Op);
231  EVT NewVT = N->getValueType(0).getVectorElementType();
232  return DAG.getNode(ISD::BITCAST, SDLoc(N),
233  NewVT, Op);
234 }
235 
236 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
237  EVT EltVT = N->getValueType(0).getVectorElementType();
238  SDValue InOp = N->getOperand(0);
239  // The BUILD_VECTOR operands may be of wider element types and
240  // we may need to truncate them back to the requested return type.
241  if (EltVT.isInteger())
242  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
243  return InOp;
244 }
245 
246 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
247  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
249  N->getOperand(0), N->getOperand(1));
250 }
251 
252 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
253  EVT NewVT = N->getValueType(0).getVectorElementType();
254  SDValue Op = GetScalarizedVector(N->getOperand(0));
255  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
256  NewVT, Op, N->getOperand(1));
257 }
258 
259 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
260  SDValue Op = GetScalarizedVector(N->getOperand(0));
261  return DAG.getNode(ISD::FPOWI, SDLoc(N),
262  Op.getValueType(), Op, N->getOperand(1));
263 }
264 
265 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
266  // The value to insert may have a wider type than the vector element type,
267  // so be sure to truncate it to the element type if necessary.
268  SDValue Op = N->getOperand(1);
269  EVT EltVT = N->getValueType(0).getVectorElementType();
270  if (Op.getValueType() != EltVT)
271  // FIXME: Can this happen for floating point types?
272  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
273  return Op;
274 }
275 
276 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
277  assert(N->isUnindexed() && "Indexed vector load?");
278 
279  SDValue Result = DAG.getLoad(
282  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
285  N->getAAInfo());
286 
287  // Legalize the chain result - switch anything that used the old chain to
288  // use the new one.
289  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
290  return Result;
291 }
292 
293 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
294  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
295  EVT DestVT = N->getValueType(0).getVectorElementType();
296  SDValue Op = N->getOperand(0);
297  EVT OpVT = Op.getValueType();
298  SDLoc DL(N);
299  // The result needs scalarizing, but it's not a given that the source does.
300  // This is a workaround for targets where it's impossible to scalarize the
301  // result of a conversion, because the source type is legal.
302  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
303  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
304  // legal and was not scalarized.
305  // See the similar logic in ScalarizeVecRes_SETCC
306  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
307  Op = GetScalarizedVector(Op);
308  } else {
309  EVT VT = OpVT.getVectorElementType();
310  Op = DAG.getNode(
311  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
312  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
313  }
314  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
315 }
316 
317 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
318  EVT EltVT = N->getValueType(0).getVectorElementType();
319  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
320  SDValue LHS = GetScalarizedVector(N->getOperand(0));
321  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
322  LHS, DAG.getValueType(ExtVT));
323 }
324 
325 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
326  SDLoc DL(N);
327  SDValue Op = N->getOperand(0);
328 
329  EVT OpVT = Op.getValueType();
330  EVT OpEltVT = OpVT.getVectorElementType();
331  EVT EltVT = N->getValueType(0).getVectorElementType();
332 
333  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
334  Op = GetScalarizedVector(Op);
335  } else {
336  Op = DAG.getNode(
337  ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
338  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
339  }
340 
341  switch (N->getOpcode()) {
343  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
345  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
347  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
348  }
349 
350  llvm_unreachable("Illegal extend_vector_inreg opcode");
351 }
352 
353 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
354  // If the operand is wider than the vector element type then it is implicitly
355  // truncated. Make that explicit here.
356  EVT EltVT = N->getValueType(0).getVectorElementType();
357  SDValue InOp = N->getOperand(0);
358  if (InOp.getValueType() != EltVT)
359  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
360  return InOp;
361 }
362 
363 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
364  SDValue Cond = N->getOperand(0);
365  EVT OpVT = Cond.getValueType();
366  SDLoc DL(N);
367  // The vselect result and true/value operands needs scalarizing, but it's
368  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
369  // See the similar logic in ScalarizeVecRes_SETCC
370  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
371  Cond = GetScalarizedVector(Cond);
372  } else {
373  EVT VT = OpVT.getVectorElementType();
374  Cond = DAG.getNode(
375  ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
376  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
377  }
378 
379  SDValue LHS = GetScalarizedVector(N->getOperand(1));
380  TargetLowering::BooleanContent ScalarBool =
381  TLI.getBooleanContents(false, false);
382  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
383 
384  // If integer and float booleans have different contents then we can't
385  // reliably optimize in all cases. There is a full explanation for this in
386  // DAGCombiner::visitSELECT() where the same issue affects folding
387  // (select C, 0, 1) to (xor C, 1).
388  if (TLI.getBooleanContents(false, false) !=
389  TLI.getBooleanContents(false, true)) {
390  // At least try the common case where the boolean is generated by a
391  // comparison.
392  if (Cond->getOpcode() == ISD::SETCC) {
393  EVT OpVT = Cond->getOperand(0).getValueType();
394  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
395  VecBool = TLI.getBooleanContents(OpVT);
396  } else
398  }
399 
400  EVT CondVT = Cond.getValueType();
401  if (ScalarBool != VecBool) {
402  switch (ScalarBool) {
404  break;
408  // Vector read from all ones, scalar expects a single 1 so mask.
409  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
410  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
411  break;
415  // Vector reads from a one, scalar from all ones so sign extend.
416  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
417  Cond, DAG.getValueType(MVT::i1));
418  break;
419  }
420  }
421 
422  // Truncate the condition if needed
423  auto BoolVT = getSetCCResultType(CondVT);
424  if (BoolVT.bitsLT(CondVT))
425  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
426 
427  return DAG.getSelect(SDLoc(N),
428  LHS.getValueType(), Cond, LHS,
429  GetScalarizedVector(N->getOperand(2)));
430 }
431 
432 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
433  SDValue LHS = GetScalarizedVector(N->getOperand(1));
434  return DAG.getSelect(SDLoc(N),
435  LHS.getValueType(), N->getOperand(0), LHS,
436  GetScalarizedVector(N->getOperand(2)));
437 }
438 
439 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
440  SDValue LHS = GetScalarizedVector(N->getOperand(2));
441  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
442  N->getOperand(0), N->getOperand(1),
443  LHS, GetScalarizedVector(N->getOperand(3)),
444  N->getOperand(4));
445 }
446 
447 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
448  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
449 }
450 
451 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
452  // Figure out if the scalar is the LHS or RHS and return it.
453  SDValue Arg = N->getOperand(2).getOperand(0);
454  if (Arg.isUndef())
455  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
456  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
457  return GetScalarizedVector(N->getOperand(Op));
458 }
459 
460 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
461  assert(N->getValueType(0).isVector() &&
462  N->getOperand(0).getValueType().isVector() &&
463  "Operand types must be vectors");
464  SDValue LHS = N->getOperand(0);
465  SDValue RHS = N->getOperand(1);
466  EVT OpVT = LHS.getValueType();
467  EVT NVT = N->getValueType(0).getVectorElementType();
468  SDLoc DL(N);
469 
470  // The result needs scalarizing, but it's not a given that the source does.
471  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
472  LHS = GetScalarizedVector(LHS);
473  RHS = GetScalarizedVector(RHS);
474  } else {
475  EVT VT = OpVT.getVectorElementType();
476  LHS = DAG.getNode(
477  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
478  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
479  RHS = DAG.getNode(
480  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
481  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
482  }
483 
484  // Turn it into a scalar SETCC.
485  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
486  N->getOperand(2));
487  // Vectors may have a different boolean contents to scalars. Promote the
488  // value appropriately.
489  ISD::NodeType ExtendCode =
491  return DAG.getNode(ExtendCode, DL, NVT, Res);
492 }
493 
494 
495 //===----------------------------------------------------------------------===//
496 // Operand Vector Scalarization <1 x ty> -> ty.
497 //===----------------------------------------------------------------------===//
498 
499 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
500  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
501  dbgs() << "\n");
502  SDValue Res = SDValue();
503 
504  if (!Res.getNode()) {
505  switch (N->getOpcode()) {
506  default:
507 #ifndef NDEBUG
508  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
509  N->dump(&DAG);
510  dbgs() << "\n";
511 #endif
512  report_fatal_error("Do not know how to scalarize this operator's "
513  "operand!\n");
514  case ISD::BITCAST:
515  Res = ScalarizeVecOp_BITCAST(N);
516  break;
517  case ISD::ANY_EXTEND:
518  case ISD::ZERO_EXTEND:
519  case ISD::SIGN_EXTEND:
520  case ISD::TRUNCATE:
521  case ISD::FP_TO_SINT:
522  case ISD::FP_TO_UINT:
523  case ISD::SINT_TO_FP:
524  case ISD::UINT_TO_FP:
525  Res = ScalarizeVecOp_UnaryOp(N);
526  break;
527  case ISD::CONCAT_VECTORS:
528  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
529  break;
531  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
532  break;
533  case ISD::VSELECT:
534  Res = ScalarizeVecOp_VSELECT(N);
535  break;
536  case ISD::SETCC:
537  Res = ScalarizeVecOp_VSETCC(N);
538  break;
539  case ISD::STORE:
540  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
541  break;
542  case ISD::FP_ROUND:
543  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
544  break;
545  }
546  }
547 
548  // If the result is null, the sub-method took care of registering results etc.
549  if (!Res.getNode()) return false;
550 
551  // If the result is N, the sub-method updated N in place. Tell the legalizer
552  // core about this.
553  if (Res.getNode() == N)
554  return true;
555 
556  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
557  "Invalid operand expansion");
558 
559  ReplaceValueWith(SDValue(N, 0), Res);
560  return false;
561 }
562 
563 /// If the value to convert is a vector that needs to be scalarized, it must be
564 /// <1 x ty>. Convert the element instead.
565 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
566  SDValue Elt = GetScalarizedVector(N->getOperand(0));
567  return DAG.getNode(ISD::BITCAST, SDLoc(N),
568  N->getValueType(0), Elt);
569 }
570 
571 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
572 /// Do the operation on the element instead.
573 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
575  "Unexpected vector type!");
576  SDValue Elt = GetScalarizedVector(N->getOperand(0));
577  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
578  N->getValueType(0).getScalarType(), Elt);
579  // Revectorize the result so the types line up with what the uses of this
580  // expression expect.
581  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
582 }
583 
584 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
585 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
587  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
588  Ops[i] = GetScalarizedVector(N->getOperand(i));
589  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
590 }
591 
592 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
593 /// so just return the element, ignoring the index.
594 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
595  EVT VT = N->getValueType(0);
596  SDValue Res = GetScalarizedVector(N->getOperand(0));
597  if (Res.getValueType() != VT)
598  Res = VT.isFloatingPoint()
599  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
600  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
601  return Res;
602 }
603 
604 /// If the input condition is a vector that needs to be scalarized, it must be
605 /// <1 x i1>, so just convert to a normal ISD::SELECT
606 /// (still with vector output type since that was acceptable if we got here).
607 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
608  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
609  EVT VT = N->getValueType(0);
610 
611  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
612  N->getOperand(2));
613 }
614 
615 /// If the operand is a vector that needs to be scalarized then the
616 /// result must be v1i1, so just convert to a scalar SETCC and wrap
617 /// with a scalar_to_vector since the res type is legal if we got here
618 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
619  assert(N->getValueType(0).isVector() &&
620  N->getOperand(0).getValueType().isVector() &&
621  "Operand types must be vectors");
622  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
623 
624  EVT VT = N->getValueType(0);
625  SDValue LHS = GetScalarizedVector(N->getOperand(0));
626  SDValue RHS = GetScalarizedVector(N->getOperand(1));
627 
628  EVT OpVT = N->getOperand(0).getValueType();
629  EVT NVT = VT.getVectorElementType();
630  SDLoc DL(N);
631  // Turn it into a scalar SETCC.
632  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
633  N->getOperand(2));
634 
635  // Vectors may have a different boolean contents to scalars. Promote the
636  // value appropriately.
637  ISD::NodeType ExtendCode =
639 
640  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
641 
642  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
643 }
644 
645 /// If the value to store is a vector that needs to be scalarized, it must be
646 /// <1 x ty>. Just store the element.
647 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
648  assert(N->isUnindexed() && "Indexed store of one-element vector?");
649  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
650  SDLoc dl(N);
651 
652  if (N->isTruncatingStore())
653  return DAG.getTruncStore(
654  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
655  N->getBasePtr(), N->getPointerInfo(),
657  N->getMemOperand()->getFlags(), N->getAAInfo());
658 
659  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
660  N->getBasePtr(), N->getPointerInfo(),
662  N->getAAInfo());
663 }
664 
665 /// If the value to round is a vector that needs to be scalarized, it must be
666 /// <1 x ty>. Convert the element instead.
667 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
668  SDValue Elt = GetScalarizedVector(N->getOperand(0));
669  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
670  N->getValueType(0).getVectorElementType(), Elt,
671  N->getOperand(1));
672  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
673 }
674 
675 //===----------------------------------------------------------------------===//
676 // Result Vector Splitting
677 //===----------------------------------------------------------------------===//
678 
679 /// This method is called when the specified result of the specified node is
680 /// found to need vector splitting. At this point, the node may also have
681 /// invalid operands or may have other results that need legalization, we just
682 /// know that (at least) one result needs vector splitting.
683 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
684  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
685  SDValue Lo, Hi;
686 
687  // See if the target wants to custom expand this node.
688  if (CustomLowerNode(N, N->getValueType(ResNo), true))
689  return;
690 
691  switch (N->getOpcode()) {
692  default:
693 #ifndef NDEBUG
694  dbgs() << "SplitVectorResult #" << ResNo << ": ";
695  N->dump(&DAG);
696  dbgs() << "\n";
697 #endif
698  report_fatal_error("Do not know how to split the result of this "
699  "operator!\n");
700 
701  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
702  case ISD::VSELECT:
703  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
704  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
705  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
706  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
707  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
708  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
709  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
710  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
711  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
712  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
713  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
714  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
715  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
716  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
717  case ISD::LOAD:
718  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
719  break;
720  case ISD::MLOAD:
721  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
722  break;
723  case ISD::MGATHER:
724  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
725  break;
726  case ISD::SETCC:
727  SplitVecRes_SETCC(N, Lo, Hi);
728  break;
729  case ISD::VECTOR_SHUFFLE:
730  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
731  break;
732 
736  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
737  break;
738 
739  case ISD::BITREVERSE:
740  case ISD::BSWAP:
741  case ISD::CTLZ:
742  case ISD::CTTZ:
745  case ISD::CTPOP:
746  case ISD::FABS:
747  case ISD::FCEIL:
748  case ISD::FCOS:
749  case ISD::FEXP:
750  case ISD::FEXP2:
751  case ISD::FFLOOR:
752  case ISD::FLOG:
753  case ISD::FLOG10:
754  case ISD::FLOG2:
755  case ISD::FNEARBYINT:
756  case ISD::FNEG:
757  case ISD::FP_EXTEND:
758  case ISD::FP_ROUND:
759  case ISD::FP_TO_SINT:
760  case ISD::FP_TO_UINT:
761  case ISD::FRINT:
762  case ISD::FROUND:
763  case ISD::FSIN:
764  case ISD::FSQRT:
765  case ISD::FTRUNC:
766  case ISD::SINT_TO_FP:
767  case ISD::TRUNCATE:
768  case ISD::UINT_TO_FP:
769  case ISD::FCANONICALIZE:
770  SplitVecRes_UnaryOp(N, Lo, Hi);
771  break;
772 
773  case ISD::ANY_EXTEND:
774  case ISD::SIGN_EXTEND:
775  case ISD::ZERO_EXTEND:
776  SplitVecRes_ExtendOp(N, Lo, Hi);
777  break;
778 
779  case ISD::ADD:
780  case ISD::SUB:
781  case ISD::MUL:
782  case ISD::MULHS:
783  case ISD::MULHU:
784  case ISD::FADD:
785  case ISD::FSUB:
786  case ISD::FMUL:
787  case ISD::FMINNUM:
788  case ISD::FMAXNUM:
789  case ISD::FMINNAN:
790  case ISD::FMAXNAN:
791  case ISD::SDIV:
792  case ISD::UDIV:
793  case ISD::FDIV:
794  case ISD::FPOW:
795  case ISD::AND:
796  case ISD::OR:
797  case ISD::XOR:
798  case ISD::SHL:
799  case ISD::SRA:
800  case ISD::SRL:
801  case ISD::UREM:
802  case ISD::SREM:
803  case ISD::FREM:
804  case ISD::SMIN:
805  case ISD::SMAX:
806  case ISD::UMIN:
807  case ISD::UMAX:
808  case ISD::SADDSAT:
809  case ISD::UADDSAT:
810  SplitVecRes_BinOp(N, Lo, Hi);
811  break;
812  case ISD::FMA:
813  SplitVecRes_TernaryOp(N, Lo, Hi);
814  break;
815  case ISD::STRICT_FADD:
816  case ISD::STRICT_FSUB:
817  case ISD::STRICT_FMUL:
818  case ISD::STRICT_FDIV:
819  case ISD::STRICT_FREM:
820  case ISD::STRICT_FSQRT:
821  case ISD::STRICT_FMA:
822  case ISD::STRICT_FPOW:
823  case ISD::STRICT_FPOWI:
824  case ISD::STRICT_FSIN:
825  case ISD::STRICT_FCOS:
826  case ISD::STRICT_FEXP:
827  case ISD::STRICT_FEXP2:
828  case ISD::STRICT_FLOG:
829  case ISD::STRICT_FLOG10:
830  case ISD::STRICT_FLOG2:
831  case ISD::STRICT_FRINT:
833  SplitVecRes_StrictFPOp(N, Lo, Hi);
834  break;
835  }
836 
837  // If Lo/Hi is null, the sub-method took care of registering results etc.
838  if (Lo.getNode())
839  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
840 }
841 
842 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
843  SDValue &Hi) {
844  SDValue LHSLo, LHSHi;
845  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
846  SDValue RHSLo, RHSHi;
847  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
848  SDLoc dl(N);
849 
850  const SDNodeFlags Flags = N->getFlags();
851  unsigned Opcode = N->getOpcode();
852  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
853  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
854 }
855 
856 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
857  SDValue &Hi) {
858  SDValue Op0Lo, Op0Hi;
859  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
860  SDValue Op1Lo, Op1Hi;
861  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
862  SDValue Op2Lo, Op2Hi;
863  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
864  SDLoc dl(N);
865 
866  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
867  Op0Lo, Op1Lo, Op2Lo);
868  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
869  Op0Hi, Op1Hi, Op2Hi);
870 }
871 
872 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
873  SDValue &Hi) {
874  // We know the result is a vector. The input may be either a vector or a
875  // scalar value.
876  EVT LoVT, HiVT;
877  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
878  SDLoc dl(N);
879 
880  SDValue InOp = N->getOperand(0);
881  EVT InVT = InOp.getValueType();
882 
883  // Handle some special cases efficiently.
884  switch (getTypeAction(InVT)) {
891  break;
894  // A scalar to vector conversion, where the scalar needs expansion.
895  // If the vector is being split in two then we can just convert the
896  // expanded pieces.
897  if (LoVT == HiVT) {
898  GetExpandedOp(InOp, Lo, Hi);
899  if (DAG.getDataLayout().isBigEndian())
900  std::swap(Lo, Hi);
901  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
902  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
903  return;
904  }
905  break;
907  // If the input is a vector that needs to be split, convert each split
908  // piece of the input now.
909  GetSplitVector(InOp, Lo, Hi);
910  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
911  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
912  return;
913  }
914 
915  // In the general case, convert the input to an integer and split it by hand.
916  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
917  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
918  if (DAG.getDataLayout().isBigEndian())
919  std::swap(LoIntVT, HiIntVT);
920 
921  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
922 
923  if (DAG.getDataLayout().isBigEndian())
924  std::swap(Lo, Hi);
925  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
926  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
927 }
928 
929 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
930  SDValue &Hi) {
931  EVT LoVT, HiVT;
932  SDLoc dl(N);
933  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
934  unsigned LoNumElts = LoVT.getVectorNumElements();
935  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
936  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
937 
938  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
939  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
940 }
941 
942 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
943  SDValue &Hi) {
944  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
945  SDLoc dl(N);
946  unsigned NumSubvectors = N->getNumOperands() / 2;
947  if (NumSubvectors == 1) {
948  Lo = N->getOperand(0);
949  Hi = N->getOperand(1);
950  return;
951  }
952 
953  EVT LoVT, HiVT;
954  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
955 
956  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
957  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
958 
959  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
960  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
961 }
962 
963 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
964  SDValue &Hi) {
965  SDValue Vec = N->getOperand(0);
966  SDValue Idx = N->getOperand(1);
967  SDLoc dl(N);
968 
969  EVT LoVT, HiVT;
970  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
971 
972  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
973  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
974  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
975  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
976  TLI.getVectorIdxTy(DAG.getDataLayout())));
977 }
978 
979 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
980  SDValue &Hi) {
981  SDValue Vec = N->getOperand(0);
982  SDValue SubVec = N->getOperand(1);
983  SDValue Idx = N->getOperand(2);
984  SDLoc dl(N);
985  GetSplitVector(Vec, Lo, Hi);
986 
987  EVT VecVT = Vec.getValueType();
988  unsigned VecElems = VecVT.getVectorNumElements();
989  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
990 
991  // If we know the index is 0, and we know the subvector doesn't cross the
992  // boundary between the halves, we can avoid spilling the vector, and insert
993  // into the lower half of the split vector directly.
994  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
995  // the index is constant and there is no boundary crossing. But those cases
996  // don't seem to get hit in practice.
997  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
998  unsigned IdxVal = ConstIdx->getZExtValue();
999  if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
1000  EVT LoVT, HiVT;
1001  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1002  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1003  return;
1004  }
1005  }
1006 
1007  // Spill the vector to the stack.
1008  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1009  SDValue Store =
1010  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1011 
1012  // Store the new subvector into the specified index.
1013  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1014  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1015  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1016  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1017 
1018  // Load the Lo part from the stack slot.
1019  Lo =
1020  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1021 
1022  // Increment the pointer to the other part.
1023  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1024  StackPtr =
1025  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1026  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
1027 
1028  // Load the Hi part from the stack slot.
1029  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1030  MinAlign(Alignment, IncrementSize));
1031 }
1032 
1033 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1034  SDValue &Hi) {
1035  SDLoc dl(N);
1036  GetSplitVector(N->getOperand(0), Lo, Hi);
1037  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1038  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1039 }
1040 
1041 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1042  SDValue &Hi) {
1043  SDValue LHSLo, LHSHi;
1044  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1045  SDLoc DL(N);
1046 
1047  SDValue RHSLo, RHSHi;
1048  SDValue RHS = N->getOperand(1);
1049  EVT RHSVT = RHS.getValueType();
1050  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1051  GetSplitVector(RHS, RHSLo, RHSHi);
1052  else
1053  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1054 
1055 
1056  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1057  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1058 }
1059 
1060 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1061  SDValue &Hi) {
1062  SDValue LHSLo, LHSHi;
1063  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1064  SDLoc dl(N);
1065 
1066  EVT LoVT, HiVT;
1067  std::tie(LoVT, HiVT) =
1068  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1069 
1070  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1071  DAG.getValueType(LoVT));
1072  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1073  DAG.getValueType(HiVT));
1074 }
1075 
1076 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1077  SDValue &Hi) {
1078  unsigned Opcode = N->getOpcode();
1079  SDValue N0 = N->getOperand(0);
1080 
1081  SDLoc dl(N);
1082  SDValue InLo, InHi;
1083 
1084  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1085  GetSplitVector(N0, InLo, InHi);
1086  else
1087  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1088 
1089  EVT InLoVT = InLo.getValueType();
1090  unsigned InNumElements = InLoVT.getVectorNumElements();
1091 
1092  EVT OutLoVT, OutHiVT;
1093  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1094  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1095  assert((2 * OutNumElements) <= InNumElements &&
1096  "Illegal extend vector in reg split");
1097 
1098  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1099  // input vector (i.e. we only use InLo):
1100  // OutLo will extend the first OutNumElements from InLo.
1101  // OutHi will extend the next OutNumElements from InLo.
1102 
1103  // Shuffle the elements from InLo for OutHi into the bottom elements to
1104  // create a 'fake' InHi.
1105  SmallVector<int, 8> SplitHi(InNumElements, -1);
1106  for (unsigned i = 0; i != OutNumElements; ++i)
1107  SplitHi[i] = i + OutNumElements;
1108  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1109 
1110  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1111  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1112 }
1113 
1114 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1115  SDValue &Hi) {
1116  unsigned NumOps = N->getNumOperands();
1117  SDValue Chain = N->getOperand(0);
1118  EVT LoVT, HiVT;
1119  SDLoc dl(N);
1120  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1121 
1124 
1125  // The Chain is the first operand.
1126  OpsLo.push_back(Chain);
1127  OpsHi.push_back(Chain);
1128 
1129  // Now process the remaining operands.
1130  for (unsigned i = 1; i < NumOps; ++i) {
1131  SDValue Op = N->getOperand(i);
1132  SDValue OpLo = Op;
1133  SDValue OpHi = Op;
1134 
1135  EVT InVT = Op.getValueType();
1136  if (InVT.isVector()) {
1137  // If the input also splits, handle it directly for a
1138  // compile time speedup. Otherwise split it by hand.
1139  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1140  GetSplitVector(Op, OpLo, OpHi);
1141  else
1142  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1143  }
1144 
1145  OpsLo.push_back(OpLo);
1146  OpsHi.push_back(OpHi);
1147  }
1148 
1149  EVT LoValueVTs[] = {LoVT, MVT::Other};
1150  EVT HiValueVTs[] = {HiVT, MVT::Other};
1151  Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1152  Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1153 
1154  // Build a factor node to remember that this Op is independent of the
1155  // other one.
1156  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1157  Lo.getValue(1), Hi.getValue(1));
1158 
1159  // Legalize the chain result - switch anything that used the old chain to
1160  // use the new one.
1161  ReplaceValueWith(SDValue(N, 1), Chain);
1162 }
1163 
1164 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1165  SDValue &Hi) {
1166  SDValue Vec = N->getOperand(0);
1167  SDValue Elt = N->getOperand(1);
1168  SDValue Idx = N->getOperand(2);
1169  SDLoc dl(N);
1170  GetSplitVector(Vec, Lo, Hi);
1171 
1172  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1173  unsigned IdxVal = CIdx->getZExtValue();
1174  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1175  if (IdxVal < LoNumElts)
1176  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1177  Lo.getValueType(), Lo, Elt, Idx);
1178  else
1179  Hi =
1180  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1181  DAG.getConstant(IdxVal - LoNumElts, dl,
1182  TLI.getVectorIdxTy(DAG.getDataLayout())));
1183  return;
1184  }
1185 
1186  // See if the target wants to custom expand this node.
1187  if (CustomLowerNode(N, N->getValueType(0), true))
1188  return;
1189 
1190  // Make the vector elements byte-addressable if they aren't already.
1191  EVT VecVT = Vec.getValueType();
1192  EVT EltVT = VecVT.getVectorElementType();
1193  if (VecVT.getScalarSizeInBits() < 8) {
1194  EltVT = MVT::i8;
1195  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1196  VecVT.getVectorNumElements());
1197  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1198  // Extend the element type to match if needed.
1199  if (EltVT.bitsGT(Elt.getValueType()))
1200  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1201  }
1202 
1203  // Spill the vector to the stack.
1204  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1205  auto &MF = DAG.getMachineFunction();
1206  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1207  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1208  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1209 
1210  // Store the new element. This may be larger than the vector element type,
1211  // so use a truncating store.
1212  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1213  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1214  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1215  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1217 
1218  EVT LoVT, HiVT;
1219  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1220 
1221  // Load the Lo part from the stack slot.
1222  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1223 
1224  // Increment the pointer to the other part.
1225  unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1226  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1227  DAG.getConstant(IncrementSize, dl,
1228  StackPtr.getValueType()));
1229 
1230  // Load the Hi part from the stack slot.
1231  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1232  PtrInfo.getWithOffset(IncrementSize),
1233  MinAlign(Alignment, IncrementSize));
1234 
1235  // If we adjusted the original type, we need to truncate the results.
1236  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1237  if (LoVT != Lo.getValueType())
1238  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1239  if (HiVT != Hi.getValueType())
1240  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1241 }
1242 
1243 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1244  SDValue &Hi) {
1245  EVT LoVT, HiVT;
1246  SDLoc dl(N);
1247  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1248  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1249  Hi = DAG.getUNDEF(HiVT);
1250 }
1251 
1252 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1253  SDValue &Hi) {
1254  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1255  EVT LoVT, HiVT;
1256  SDLoc dl(LD);
1257  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1258 
1260  SDValue Ch = LD->getChain();
1261  SDValue Ptr = LD->getBasePtr();
1262  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1263  EVT MemoryVT = LD->getMemoryVT();
1264  unsigned Alignment = LD->getOriginalAlignment();
1265  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1266  AAMDNodes AAInfo = LD->getAAInfo();
1267 
1268  EVT LoMemVT, HiMemVT;
1269  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1270 
1271  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1272  LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1273 
1274  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1275  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1276  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1277  LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1278  Alignment, MMOFlags, AAInfo);
1279 
1280  // Build a factor node to remember that this load is independent of the
1281  // other one.
1282  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1283  Hi.getValue(1));
1284 
1285  // Legalize the chain result - switch anything that used the old chain to
1286  // use the new one.
1287  ReplaceValueWith(SDValue(LD, 1), Ch);
1288 }
1289 
1290 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1291  SDValue &Lo, SDValue &Hi) {
1292  EVT LoVT, HiVT;
1293  SDLoc dl(MLD);
1294  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1295 
1296  SDValue Ch = MLD->getChain();
1297  SDValue Ptr = MLD->getBasePtr();
1298  SDValue Mask = MLD->getMask();
1299  SDValue PassThru = MLD->getPassThru();
1300  unsigned Alignment = MLD->getOriginalAlignment();
1302 
1303  // if Alignment is equal to the vector size,
1304  // take the half of it for the second part
1305  unsigned SecondHalfAlignment =
1306  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1307  Alignment/2 : Alignment;
1308 
1309  // Split Mask operand
1310  SDValue MaskLo, MaskHi;
1311  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1312  GetSplitVector(Mask, MaskLo, MaskHi);
1313  else
1314  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1315 
1316  EVT MemoryVT = MLD->getMemoryVT();
1317  EVT LoMemVT, HiMemVT;
1318  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1319 
1320  SDValue PassThruLo, PassThruHi;
1321  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1322  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1323  else
1324  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1325 
1327  getMachineMemOperand(MLD->getPointerInfo(),
1329  Alignment, MLD->getAAInfo(), MLD->getRanges());
1330 
1331  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
1332  ExtType, MLD->isExpandingLoad());
1333 
1334  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1335  MLD->isExpandingLoad());
1336  unsigned HiOffset = LoMemVT.getStoreSize();
1337 
1340  HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
1341  MLD->getRanges());
1342 
1343  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
1344  ExtType, MLD->isExpandingLoad());
1345 
1346  // Build a factor node to remember that this load is independent of the
1347  // other one.
1348  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1349  Hi.getValue(1));
1350 
1351  // Legalize the chain result - switch anything that used the old chain to
1352  // use the new one.
1353  ReplaceValueWith(SDValue(MLD, 1), Ch);
1354 
1355 }
1356 
1357 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1358  SDValue &Lo, SDValue &Hi) {
1359  EVT LoVT, HiVT;
1360  SDLoc dl(MGT);
1361  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1362 
1363  SDValue Ch = MGT->getChain();
1364  SDValue Ptr = MGT->getBasePtr();
1365  SDValue Mask = MGT->getMask();
1366  SDValue PassThru = MGT->getPassThru();
1367  SDValue Index = MGT->getIndex();
1368  SDValue Scale = MGT->getScale();
1369  unsigned Alignment = MGT->getOriginalAlignment();
1370 
1371  // Split Mask operand
1372  SDValue MaskLo, MaskHi;
1373  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1374  GetSplitVector(Mask, MaskLo, MaskHi);
1375  else
1376  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1377 
1378  EVT MemoryVT = MGT->getMemoryVT();
1379  EVT LoMemVT, HiMemVT;
1380  // Split MemoryVT
1381  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1382 
1383  SDValue PassThruLo, PassThruHi;
1384  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1385  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1386  else
1387  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1388 
1389  SDValue IndexHi, IndexLo;
1390  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1391  GetSplitVector(Index, IndexLo, IndexHi);
1392  else
1393  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1394 
1396  getMachineMemOperand(MGT->getPointerInfo(),
1398  Alignment, MGT->getAAInfo(), MGT->getRanges());
1399 
1400  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1401  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1402  MMO);
1403 
1404  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1405  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1406  MMO);
1407 
1408  // Build a factor node to remember that this load is independent of the
1409  // other one.
1410  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1411  Hi.getValue(1));
1412 
1413  // Legalize the chain result - switch anything that used the old chain to
1414  // use the new one.
1415  ReplaceValueWith(SDValue(MGT, 1), Ch);
1416 }
1417 
1418 
1419 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1420  assert(N->getValueType(0).isVector() &&
1421  N->getOperand(0).getValueType().isVector() &&
1422  "Operand types must be vectors");
1423 
1424  EVT LoVT, HiVT;
1425  SDLoc DL(N);
1426  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1427 
1428  // If the input also splits, handle it directly. Otherwise split it by hand.
1429  SDValue LL, LH, RL, RH;
1430  if (getTypeAction(N->getOperand(0).getValueType()) ==
1432  GetSplitVector(N->getOperand(0), LL, LH);
1433  else
1434  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1435 
1436  if (getTypeAction(N->getOperand(1).getValueType()) ==
1438  GetSplitVector(N->getOperand(1), RL, RH);
1439  else
1440  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1441 
1442  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1443  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1444 }
1445 
1446 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1447  SDValue &Hi) {
1448  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1449  EVT LoVT, HiVT;
1450  SDLoc dl(N);
1451  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1452 
1453  // If the input also splits, handle it directly for a compile time speedup.
1454  // Otherwise split it by hand.
1455  EVT InVT = N->getOperand(0).getValueType();
1456  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1457  GetSplitVector(N->getOperand(0), Lo, Hi);
1458  else
1459  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1460 
1461  if (N->getOpcode() == ISD::FP_ROUND) {
1462  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1463  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1464  } else {
1465  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1466  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1467  }
1468 }
1469 
1470 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1471  SDValue &Hi) {
1472  SDLoc dl(N);
1473  EVT SrcVT = N->getOperand(0).getValueType();
1474  EVT DestVT = N->getValueType(0);
1475  EVT LoVT, HiVT;
1476  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1477 
1478  // We can do better than a generic split operation if the extend is doing
1479  // more than just doubling the width of the elements and the following are
1480  // true:
1481  // - The number of vector elements is even,
1482  // - the source type is legal,
1483  // - the type of a split source is illegal,
1484  // - the type of an extended (by doubling element size) source is legal, and
1485  // - the type of that extended source when split is legal.
1486  //
1487  // This won't necessarily completely legalize the operation, but it will
1488  // more effectively move in the right direction and prevent falling down
1489  // to scalarization in many cases due to the input vector being split too
1490  // far.
1491  unsigned NumElements = SrcVT.getVectorNumElements();
1492  if ((NumElements & 1) == 0 &&
1493  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1494  LLVMContext &Ctx = *DAG.getContext();
1495  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1496  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1497 
1498  EVT SplitLoVT, SplitHiVT;
1499  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1500  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1501  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1502  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1503  N->dump(&DAG); dbgs() << "\n");
1504  // Extend the source vector by one step.
1505  SDValue NewSrc =
1506  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1507  // Get the low and high halves of the new, extended one step, vector.
1508  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1509  // Extend those vector halves the rest of the way.
1510  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1511  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1512  return;
1513  }
1514  }
1515  // Fall back to the generic unary operator splitting otherwise.
1516  SplitVecRes_UnaryOp(N, Lo, Hi);
1517 }
1518 
1519 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1520  SDValue &Lo, SDValue &Hi) {
1521  // The low and high parts of the original input give four input vectors.
1522  SDValue Inputs[4];
1523  SDLoc dl(N);
1524  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1525  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1526  EVT NewVT = Inputs[0].getValueType();
1527  unsigned NewElts = NewVT.getVectorNumElements();
1528 
1529  // If Lo or Hi uses elements from at most two of the four input vectors, then
1530  // express it as a vector shuffle of those two inputs. Otherwise extract the
1531  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1533  for (unsigned High = 0; High < 2; ++High) {
1534  SDValue &Output = High ? Hi : Lo;
1535 
1536  // Build a shuffle mask for the output, discovering on the fly which
1537  // input vectors to use as shuffle operands (recorded in InputUsed).
1538  // If building a suitable shuffle vector proves too hard, then bail
1539  // out with useBuildVector set.
1540  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1541  unsigned FirstMaskIdx = High * NewElts;
1542  bool useBuildVector = false;
1543  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1544  // The mask element. This indexes into the input.
1545  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1546 
1547  // The input vector this mask element indexes into.
1548  unsigned Input = (unsigned)Idx / NewElts;
1549 
1550  if (Input >= array_lengthof(Inputs)) {
1551  // The mask element does not index into any input vector.
1552  Ops.push_back(-1);
1553  continue;
1554  }
1555 
1556  // Turn the index into an offset from the start of the input vector.
1557  Idx -= Input * NewElts;
1558 
1559  // Find or create a shuffle vector operand to hold this input.
1560  unsigned OpNo;
1561  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1562  if (InputUsed[OpNo] == Input) {
1563  // This input vector is already an operand.
1564  break;
1565  } else if (InputUsed[OpNo] == -1U) {
1566  // Create a new operand for this input vector.
1567  InputUsed[OpNo] = Input;
1568  break;
1569  }
1570  }
1571 
1572  if (OpNo >= array_lengthof(InputUsed)) {
1573  // More than two input vectors used! Give up on trying to create a
1574  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1575  useBuildVector = true;
1576  break;
1577  }
1578 
1579  // Add the mask index for the new shuffle vector.
1580  Ops.push_back(Idx + OpNo * NewElts);
1581  }
1582 
1583  if (useBuildVector) {
1584  EVT EltVT = NewVT.getVectorElementType();
1586 
1587  // Extract the input elements by hand.
1588  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1589  // The mask element. This indexes into the input.
1590  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1591 
1592  // The input vector this mask element indexes into.
1593  unsigned Input = (unsigned)Idx / NewElts;
1594 
1595  if (Input >= array_lengthof(Inputs)) {
1596  // The mask element is "undef" or indexes off the end of the input.
1597  SVOps.push_back(DAG.getUNDEF(EltVT));
1598  continue;
1599  }
1600 
1601  // Turn the index into an offset from the start of the input vector.
1602  Idx -= Input * NewElts;
1603 
1604  // Extract the vector element by hand.
1605  SVOps.push_back(DAG.getNode(
1606  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1607  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1608  }
1609 
1610  // Construct the Lo/Hi output using a BUILD_VECTOR.
1611  Output = DAG.getBuildVector(NewVT, dl, SVOps);
1612  } else if (InputUsed[0] == -1U) {
1613  // No input vectors were used! The result is undefined.
1614  Output = DAG.getUNDEF(NewVT);
1615  } else {
1616  SDValue Op0 = Inputs[InputUsed[0]];
1617  // If only one input was used, use an undefined vector for the other.
1618  SDValue Op1 = InputUsed[1] == -1U ?
1619  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1620  // At least one input vector was used. Create a new shuffle vector.
1621  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1622  }
1623 
1624  Ops.clear();
1625  }
1626 }
1627 
1628 
1629 //===----------------------------------------------------------------------===//
1630 // Operand Vector Splitting
1631 //===----------------------------------------------------------------------===//
1632 
1633 /// This method is called when the specified operand of the specified node is
1634 /// found to need vector splitting. At this point, all of the result types of
1635 /// the node are known to be legal, but other operands of the node may need
1636 /// legalization as well as the specified one.
1637 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1638  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1639  SDValue Res = SDValue();
1640 
1641  // See if the target wants to custom split this node.
1642  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1643  return false;
1644 
1645  if (!Res.getNode()) {
1646  switch (N->getOpcode()) {
1647  default:
1648 #ifndef NDEBUG
1649  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1650  N->dump(&DAG);
1651  dbgs() << "\n";
1652 #endif
1653  report_fatal_error("Do not know how to split this operator's "
1654  "operand!\n");
1655 
1656  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1657  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1658  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1659  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1660  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1661  case ISD::TRUNCATE:
1662  Res = SplitVecOp_TruncateHelper(N);
1663  break;
1664  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1665  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1666  case ISD::STORE:
1667  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1668  break;
1669  case ISD::MSTORE:
1670  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1671  break;
1672  case ISD::MSCATTER:
1673  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1674  break;
1675  case ISD::MGATHER:
1676  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1677  break;
1678  case ISD::VSELECT:
1679  Res = SplitVecOp_VSELECT(N, OpNo);
1680  break;
1681  case ISD::FP_TO_SINT:
1682  case ISD::FP_TO_UINT:
1683  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1684  Res = SplitVecOp_TruncateHelper(N);
1685  else
1686  Res = SplitVecOp_UnaryOp(N);
1687  break;
1688  case ISD::SINT_TO_FP:
1689  case ISD::UINT_TO_FP:
1690  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1691  Res = SplitVecOp_TruncateHelper(N);
1692  else
1693  Res = SplitVecOp_UnaryOp(N);
1694  break;
1695  case ISD::CTTZ:
1696  case ISD::CTLZ:
1697  case ISD::CTPOP:
1698  case ISD::FP_EXTEND:
1699  case ISD::SIGN_EXTEND:
1700  case ISD::ZERO_EXTEND:
1701  case ISD::ANY_EXTEND:
1702  case ISD::FTRUNC:
1703  case ISD::FCANONICALIZE:
1704  Res = SplitVecOp_UnaryOp(N);
1705  break;
1706 
1710  Res = SplitVecOp_ExtVecInRegOp(N);
1711  break;
1712 
1713  case ISD::VECREDUCE_FADD:
1714  case ISD::VECREDUCE_FMUL:
1715  case ISD::VECREDUCE_ADD:
1716  case ISD::VECREDUCE_MUL:
1717  case ISD::VECREDUCE_AND:
1718  case ISD::VECREDUCE_OR:
1719  case ISD::VECREDUCE_XOR:
1720  case ISD::VECREDUCE_SMAX:
1721  case ISD::VECREDUCE_SMIN:
1722  case ISD::VECREDUCE_UMAX:
1723  case ISD::VECREDUCE_UMIN:
1724  case ISD::VECREDUCE_FMAX:
1725  case ISD::VECREDUCE_FMIN:
1726  Res = SplitVecOp_VECREDUCE(N, OpNo);
1727  break;
1728  }
1729  }
1730 
1731  // If the result is null, the sub-method took care of registering results etc.
1732  if (!Res.getNode()) return false;
1733 
1734  // If the result is N, the sub-method updated N in place. Tell the legalizer
1735  // core about this.
1736  if (Res.getNode() == N)
1737  return true;
1738 
1739  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1740  "Invalid operand expansion");
1741 
1742  ReplaceValueWith(SDValue(N, 0), Res);
1743  return false;
1744 }
1745 
1746 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1747  // The only possibility for an illegal operand is the mask, since result type
1748  // legalization would have handled this node already otherwise.
1749  assert(OpNo == 0 && "Illegal operand must be mask");
1750 
1751  SDValue Mask = N->getOperand(0);
1752  SDValue Src0 = N->getOperand(1);
1753  SDValue Src1 = N->getOperand(2);
1754  EVT Src0VT = Src0.getValueType();
1755  SDLoc DL(N);
1756  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1757 
1758  SDValue Lo, Hi;
1759  GetSplitVector(N->getOperand(0), Lo, Hi);
1760  assert(Lo.getValueType() == Hi.getValueType() &&
1761  "Lo and Hi have differing types");
1762 
1763  EVT LoOpVT, HiOpVT;
1764  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1765  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1766 
1767  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1768  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1769  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1770  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1771 
1772  SDValue LoSelect =
1773  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1774  SDValue HiSelect =
1775  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1776 
1777  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1778 }
1779 
1780 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
1781  EVT ResVT = N->getValueType(0);
1782  SDValue Lo, Hi;
1783  SDLoc dl(N);
1784 
1785  SDValue VecOp = N->getOperand(OpNo);
1786  EVT VecVT = VecOp.getValueType();
1787  assert(VecVT.isVector() && "Can only split reduce vector operand");
1788  GetSplitVector(VecOp, Lo, Hi);
1789  EVT LoOpVT, HiOpVT;
1790  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
1791 
1792  bool NoNaN = N->getFlags().hasNoNaNs();
1793  unsigned CombineOpc = 0;
1794  switch (N->getOpcode()) {
1795  case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
1796  case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
1797  case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
1798  case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
1799  case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
1800  case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
1801  case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
1802  case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
1803  case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
1804  case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
1805  case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
1806  case ISD::VECREDUCE_FMAX:
1807  CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
1808  break;
1809  case ISD::VECREDUCE_FMIN:
1810  CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
1811  break;
1812  default:
1813  llvm_unreachable("Unexpected reduce ISD node");
1814  }
1815 
1816  // Use the appropriate scalar instruction on the split subvectors before
1817  // reducing the now partially reduced smaller vector.
1818  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
1819  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
1820 }
1821 
1822 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1823  // The result has a legal vector type, but the input needs splitting.
1824  EVT ResVT = N->getValueType(0);
1825  SDValue Lo, Hi;
1826  SDLoc dl(N);
1827  GetSplitVector(N->getOperand(0), Lo, Hi);
1828  EVT InVT = Lo.getValueType();
1829 
1830  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1831  InVT.getVectorNumElements());
1832 
1833  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1834  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1835 
1836  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1837 }
1838 
1839 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1840  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1841  // end up being split all the way down to individual components. Convert the
1842  // split pieces into integers and reassemble.
1843  SDValue Lo, Hi;
1844  GetSplitVector(N->getOperand(0), Lo, Hi);
1845  Lo = BitConvertToInteger(Lo);
1846  Hi = BitConvertToInteger(Hi);
1847 
1848  if (DAG.getDataLayout().isBigEndian())
1849  std::swap(Lo, Hi);
1850 
1851  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1852  JoinIntegers(Lo, Hi));
1853 }
1854 
1855 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1856  // We know that the extracted result type is legal.
1857  EVT SubVT = N->getValueType(0);
1858  SDValue Idx = N->getOperand(1);
1859  SDLoc dl(N);
1860  SDValue Lo, Hi;
1861  GetSplitVector(N->getOperand(0), Lo, Hi);
1862 
1863  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1864  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1865 
1866  if (IdxVal < LoElts) {
1867  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1868  "Extracted subvector crosses vector split!");
1869  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1870  } else {
1871  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1872  DAG.getConstant(IdxVal - LoElts, dl,
1873  Idx.getValueType()));
1874  }
1875 }
1876 
1877 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1878  SDValue Vec = N->getOperand(0);
1879  SDValue Idx = N->getOperand(1);
1880  EVT VecVT = Vec.getValueType();
1881 
1882  if (isa<ConstantSDNode>(Idx)) {
1883  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1884  assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1885 
1886  SDValue Lo, Hi;
1887  GetSplitVector(Vec, Lo, Hi);
1888 
1889  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1890 
1891  if (IdxVal < LoElts)
1892  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1893  return SDValue(DAG.UpdateNodeOperands(N, Hi,
1894  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
1895  Idx.getValueType())), 0);
1896  }
1897 
1898  // See if the target wants to custom expand this node.
1899  if (CustomLowerNode(N, N->getValueType(0), true))
1900  return SDValue();
1901 
1902  // Make the vector elements byte-addressable if they aren't already.
1903  SDLoc dl(N);
1904  EVT EltVT = VecVT.getVectorElementType();
1905  if (VecVT.getScalarSizeInBits() < 8) {
1906  EltVT = MVT::i8;
1907  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1908  VecVT.getVectorNumElements());
1909  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1910  }
1911 
1912  // Store the vector to the stack.
1913  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1914  auto &MF = DAG.getMachineFunction();
1915  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1916  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1917  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1918 
1919  // Load back the required element.
1920  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1921  return DAG.getExtLoad(
1922  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1924 }
1925 
1926 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
1927  SDValue Lo, Hi;
1928 
1929  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
1930  // splitting the result has the same effect as splitting the input operand.
1931  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1932 
1933  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
1934 }
1935 
1936 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
1937  unsigned OpNo) {
1938  EVT LoVT, HiVT;
1939  SDLoc dl(MGT);
1940  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1941 
1942  SDValue Ch = MGT->getChain();
1943  SDValue Ptr = MGT->getBasePtr();
1944  SDValue Index = MGT->getIndex();
1945  SDValue Scale = MGT->getScale();
1946  SDValue Mask = MGT->getMask();
1947  SDValue PassThru = MGT->getPassThru();
1948  unsigned Alignment = MGT->getOriginalAlignment();
1949 
1950  SDValue MaskLo, MaskHi;
1951  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1952  // Split Mask operand
1953  GetSplitVector(Mask, MaskLo, MaskHi);
1954  else
1955  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1956 
1957  EVT MemoryVT = MGT->getMemoryVT();
1958  EVT LoMemVT, HiMemVT;
1959  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1960 
1961  SDValue PassThruLo, PassThruHi;
1962  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1963  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1964  else
1965  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1966 
1967  SDValue IndexHi, IndexLo;
1968  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1969  GetSplitVector(Index, IndexLo, IndexHi);
1970  else
1971  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1972 
1974  getMachineMemOperand(MGT->getPointerInfo(),
1976  Alignment, MGT->getAAInfo(), MGT->getRanges());
1977 
1978  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1979  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
1980  OpsLo, MMO);
1981 
1982  MMO = DAG.getMachineFunction().
1983  getMachineMemOperand(MGT->getPointerInfo(),
1985  Alignment, MGT->getAAInfo(),
1986  MGT->getRanges());
1987 
1988  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1989  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
1990  OpsHi, MMO);
1991 
1992  // Build a factor node to remember that this load is independent of the
1993  // other one.
1994  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1995  Hi.getValue(1));
1996 
1997  // Legalize the chain result - switch anything that used the old chain to
1998  // use the new one.
1999  ReplaceValueWith(SDValue(MGT, 1), Ch);
2000 
2001  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2002  Hi);
2003  ReplaceValueWith(SDValue(MGT, 0), Res);
2004  return SDValue();
2005 }
2006 
2007 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2008  unsigned OpNo) {
2009  SDValue Ch = N->getChain();
2010  SDValue Ptr = N->getBasePtr();
2011  SDValue Mask = N->getMask();
2012  SDValue Data = N->getValue();
2013  EVT MemoryVT = N->getMemoryVT();
2014  unsigned Alignment = N->getOriginalAlignment();
2015  SDLoc DL(N);
2016 
2017  EVT LoMemVT, HiMemVT;
2018  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2019 
2020  SDValue DataLo, DataHi;
2021  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2022  // Split Data operand
2023  GetSplitVector(Data, DataLo, DataHi);
2024  else
2025  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2026 
2027  SDValue MaskLo, MaskHi;
2028  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2029  // Split Mask operand
2030  GetSplitVector(Mask, MaskLo, MaskHi);
2031  else
2032  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2033 
2034  // if Alignment is equal to the vector size,
2035  // take the half of it for the second part
2036  unsigned SecondHalfAlignment =
2037  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
2038  Alignment/2 : Alignment;
2039 
2040  SDValue Lo, Hi;
2042  getMachineMemOperand(N->getPointerInfo(),
2044  Alignment, N->getAAInfo(), N->getRanges());
2045 
2046  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
2047  N->isTruncatingStore(),
2048  N->isCompressingStore());
2049 
2050  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2051  N->isCompressingStore());
2052  unsigned HiOffset = LoMemVT.getStoreSize();
2053 
2056  HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
2057  N->getRanges());
2058 
2059  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
2061 
2062  // Build a factor node to remember that this store is independent of the
2063  // other one.
2064  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2065 }
2066 
2067 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2068  unsigned OpNo) {
2069  SDValue Ch = N->getChain();
2070  SDValue Ptr = N->getBasePtr();
2071  SDValue Mask = N->getMask();
2072  SDValue Index = N->getIndex();
2073  SDValue Scale = N->getScale();
2074  SDValue Data = N->getValue();
2075  EVT MemoryVT = N->getMemoryVT();
2076  unsigned Alignment = N->getOriginalAlignment();
2077  SDLoc DL(N);
2078 
2079  // Split all operands
2080  EVT LoMemVT, HiMemVT;
2081  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2082 
2083  SDValue DataLo, DataHi;
2084  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2085  // Split Data operand
2086  GetSplitVector(Data, DataLo, DataHi);
2087  else
2088  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2089 
2090  SDValue MaskLo, MaskHi;
2091  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2092  // Split Mask operand
2093  GetSplitVector(Mask, MaskLo, MaskHi);
2094  else
2095  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2096 
2097  SDValue IndexHi, IndexLo;
2098  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2099  GetSplitVector(Index, IndexLo, IndexHi);
2100  else
2101  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2102 
2103  SDValue Lo;
2105  getMachineMemOperand(N->getPointerInfo(),
2107  Alignment, N->getAAInfo(), N->getRanges());
2108 
2109  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2110  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2111  DL, OpsLo, MMO);
2112 
2113  MMO = DAG.getMachineFunction().
2114  getMachineMemOperand(N->getPointerInfo(),
2116  Alignment, N->getAAInfo(), N->getRanges());
2117 
2118  // The order of the Scatter operation after split is well defined. The "Hi"
2119  // part comes after the "Lo". So these two operations should be chained one
2120  // after another.
2121  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2122  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2123  DL, OpsHi, MMO);
2124 }
2125 
2126 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2127  assert(N->isUnindexed() && "Indexed store of vector?");
2128  assert(OpNo == 1 && "Can only split the stored value");
2129  SDLoc DL(N);
2130 
2131  bool isTruncating = N->isTruncatingStore();
2132  SDValue Ch = N->getChain();
2133  SDValue Ptr = N->getBasePtr();
2134  EVT MemoryVT = N->getMemoryVT();
2135  unsigned Alignment = N->getOriginalAlignment();
2136  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2137  AAMDNodes AAInfo = N->getAAInfo();
2138  SDValue Lo, Hi;
2139  GetSplitVector(N->getOperand(1), Lo, Hi);
2140 
2141  EVT LoMemVT, HiMemVT;
2142  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2143 
2144  // Scalarize if the split halves are not byte-sized.
2145  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2146  return TLI.scalarizeVectorStore(N, DAG);
2147 
2148  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2149 
2150  if (isTruncating)
2151  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2152  Alignment, MMOFlags, AAInfo);
2153  else
2154  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2155  AAInfo);
2156 
2157  // Increment the pointer to the other half.
2158  Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2159 
2160  if (isTruncating)
2161  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2162  N->getPointerInfo().getWithOffset(IncrementSize),
2163  HiMemVT, Alignment, MMOFlags, AAInfo);
2164  else
2165  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2166  N->getPointerInfo().getWithOffset(IncrementSize),
2167  Alignment, MMOFlags, AAInfo);
2168 
2169  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2170 }
2171 
2172 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2173  SDLoc DL(N);
2174 
2175  // The input operands all must have the same type, and we know the result
2176  // type is valid. Convert this to a buildvector which extracts all the
2177  // input elements.
2178  // TODO: If the input elements are power-two vectors, we could convert this to
2179  // a new CONCAT_VECTORS node with elements that are half-wide.
2181  EVT EltVT = N->getValueType(0).getVectorElementType();
2182  for (const SDValue &Op : N->op_values()) {
2183  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2184  i != e; ++i) {
2185  Elts.push_back(DAG.getNode(
2186  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2187  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2188  }
2189  }
2190 
2191  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2192 }
2193 
2194 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2195  // The result type is legal, but the input type is illegal. If splitting
2196  // ends up with the result type of each half still being legal, just
2197  // do that. If, however, that would result in an illegal result type,
2198  // we can try to get more clever with power-two vectors. Specifically,
2199  // split the input type, but also widen the result element size, then
2200  // concatenate the halves and truncate again. For example, consider a target
2201  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2202  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2203  // %inlo = v4i32 extract_subvector %in, 0
2204  // %inhi = v4i32 extract_subvector %in, 4
2205  // %lo16 = v4i16 trunc v4i32 %inlo
2206  // %hi16 = v4i16 trunc v4i32 %inhi
2207  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2208  // %res = v8i8 trunc v8i16 %in16
2209  //
2210  // Without this transform, the original truncate would end up being
2211  // scalarized, which is pretty much always a last resort.
2212  SDValue InVec = N->getOperand(0);
2213  EVT InVT = InVec->getValueType(0);
2214  EVT OutVT = N->getValueType(0);
2215  unsigned NumElements = OutVT.getVectorNumElements();
2216  bool IsFloat = OutVT.isFloatingPoint();
2217 
2218  // Widening should have already made sure this is a power-two vector
2219  // if we're trying to split it at all. assert() that's true, just in case.
2220  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2221 
2222  unsigned InElementSize = InVT.getScalarSizeInBits();
2223  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2224 
2225  // If the input elements are only 1/2 the width of the result elements,
2226  // just use the normal splitting. Our trick only work if there's room
2227  // to split more than once.
2228  if (InElementSize <= OutElementSize * 2)
2229  return SplitVecOp_UnaryOp(N);
2230  SDLoc DL(N);
2231 
2232  // Get the split input vector.
2233  SDValue InLoVec, InHiVec;
2234  GetSplitVector(InVec, InLoVec, InHiVec);
2235  // Truncate them to 1/2 the element size.
2236  EVT HalfElementVT = IsFloat ?
2237  EVT::getFloatingPointVT(InElementSize/2) :
2238  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2239  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2240  NumElements/2);
2241  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2242  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2243  // Concatenate them to get the full intermediate truncation result.
2244  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2245  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2246  HalfHi);
2247  // Now finish up by truncating all the way down to the original result
2248  // type. This should normally be something that ends up being legal directly,
2249  // but in theory if a target has very wide vectors and an annoyingly
2250  // restricted set of legal types, this split can chain to build things up.
2251  return IsFloat
2252  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2253  DAG.getTargetConstant(
2254  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2255  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2256 }
2257 
2258 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2259  assert(N->getValueType(0).isVector() &&
2260  N->getOperand(0).getValueType().isVector() &&
2261  "Operand types must be vectors");
2262  // The result has a legal vector type, but the input needs splitting.
2263  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2264  SDLoc DL(N);
2265  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2266  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2267  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2268  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2269  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2270 
2271  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2272  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2273  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2274  return PromoteTargetBoolean(Con, N->getValueType(0));
2275 }
2276 
2277 
2278 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2279  // The result has a legal vector type, but the input needs splitting.
2280  EVT ResVT = N->getValueType(0);
2281  SDValue Lo, Hi;
2282  SDLoc DL(N);
2283  GetSplitVector(N->getOperand(0), Lo, Hi);
2284  EVT InVT = Lo.getValueType();
2285 
2286  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2287  InVT.getVectorNumElements());
2288 
2289  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2290  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2291 
2292  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2293 }
2294 
2295 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2296  // The result (and the first input) has a legal vector type, but the second
2297  // input needs splitting.
2298  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2299 }
2300 
2301 
2302 //===----------------------------------------------------------------------===//
2303 // Result Vector Widening
2304 //===----------------------------------------------------------------------===//
2305 
2306 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2307  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2308  dbgs() << "\n");
2309 
2310  // See if the target wants to custom widen this node.
2311  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2312  return;
2313 
2314  SDValue Res = SDValue();
2315  switch (N->getOpcode()) {
2316  default:
2317 #ifndef NDEBUG
2318  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2319  N->dump(&DAG);
2320  dbgs() << "\n";
2321 #endif
2322  llvm_unreachable("Do not know how to widen the result of this operator!");
2323 
2324  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2325  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2326  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2327  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2328  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2329  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2330  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2331  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2332  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2333  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2334  case ISD::VSELECT:
2335  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2336  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2337  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2338  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2339  case ISD::VECTOR_SHUFFLE:
2340  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2341  break;
2342  case ISD::MLOAD:
2343  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2344  break;
2345  case ISD::MGATHER:
2346  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2347  break;
2348 
2349  case ISD::ADD:
2350  case ISD::AND:
2351  case ISD::MUL:
2352  case ISD::MULHS:
2353  case ISD::MULHU:
2354  case ISD::OR:
2355  case ISD::SUB:
2356  case ISD::XOR:
2357  case ISD::FMINNUM:
2358  case ISD::FMAXNUM:
2359  case ISD::FMINNAN:
2360  case ISD::FMAXNAN:
2361  case ISD::SMIN:
2362  case ISD::SMAX:
2363  case ISD::UMIN:
2364  case ISD::UMAX:
2365  Res = WidenVecRes_Binary(N);
2366  break;
2367 
2368  case ISD::FADD:
2369  case ISD::FMUL:
2370  case ISD::FPOW:
2371  case ISD::FSUB:
2372  case ISD::FDIV:
2373  case ISD::FREM:
2374  case ISD::SDIV:
2375  case ISD::UDIV:
2376  case ISD::SREM:
2377  case ISD::UREM:
2378  Res = WidenVecRes_BinaryCanTrap(N);
2379  break;
2380 
2381  case ISD::STRICT_FADD:
2382  case ISD::STRICT_FSUB:
2383  case ISD::STRICT_FMUL:
2384  case ISD::STRICT_FDIV:
2385  case ISD::STRICT_FREM:
2386  case ISD::STRICT_FSQRT:
2387  case ISD::STRICT_FMA:
2388  case ISD::STRICT_FPOW:
2389  case ISD::STRICT_FPOWI:
2390  case ISD::STRICT_FSIN:
2391  case ISD::STRICT_FCOS:
2392  case ISD::STRICT_FEXP:
2393  case ISD::STRICT_FEXP2:
2394  case ISD::STRICT_FLOG:
2395  case ISD::STRICT_FLOG10:
2396  case ISD::STRICT_FLOG2:
2397  case ISD::STRICT_FRINT:
2399  Res = WidenVecRes_StrictFP(N);
2400  break;
2401 
2402  case ISD::FCOPYSIGN:
2403  Res = WidenVecRes_FCOPYSIGN(N);
2404  break;
2405 
2406  case ISD::FPOWI:
2407  Res = WidenVecRes_POWI(N);
2408  break;
2409 
2410  case ISD::SHL:
2411  case ISD::SRA:
2412  case ISD::SRL:
2413  Res = WidenVecRes_Shift(N);
2414  break;
2415 
2419  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2420  break;
2421 
2422  case ISD::ANY_EXTEND:
2423  case ISD::FP_EXTEND:
2424  case ISD::FP_ROUND:
2425  case ISD::FP_TO_SINT:
2426  case ISD::FP_TO_UINT:
2427  case ISD::SIGN_EXTEND:
2428  case ISD::SINT_TO_FP:
2429  case ISD::TRUNCATE:
2430  case ISD::UINT_TO_FP:
2431  case ISD::ZERO_EXTEND:
2432  Res = WidenVecRes_Convert(N);
2433  break;
2434 
2435  case ISD::FABS:
2436  case ISD::FCEIL:
2437  case ISD::FCOS:
2438  case ISD::FEXP:
2439  case ISD::FEXP2:
2440  case ISD::FFLOOR:
2441  case ISD::FLOG:
2442  case ISD::FLOG10:
2443  case ISD::FLOG2:
2444  case ISD::FNEARBYINT:
2445  case ISD::FRINT:
2446  case ISD::FROUND:
2447  case ISD::FSIN:
2448  case ISD::FSQRT:
2449  case ISD::FTRUNC: {
2450  // We're going to widen this vector op to a legal type by padding with undef
2451  // elements. If the wide vector op is eventually going to be expanded to
2452  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2453  // libcalls on the undef elements. We are assuming that if the scalar op
2454  // requires expanding, then the vector op needs expanding too.
2455  EVT VT = N->getValueType(0);
2456  if (TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
2457  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2458  assert(!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2459  "Target supports vector op, but scalar requires expansion?");
2460  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2461  break;
2462  }
2463  }
2464  // If the target has custom/legal support for the scalar FP intrinsic ops
2465  // (they are probably not destined to become libcalls), then widen those like
2466  // any other unary ops.
2468 
2469  case ISD::BITREVERSE:
2470  case ISD::BSWAP:
2471  case ISD::CTLZ:
2472  case ISD::CTPOP:
2473  case ISD::CTTZ:
2474  case ISD::FNEG:
2475  case ISD::FCANONICALIZE:
2476  Res = WidenVecRes_Unary(N);
2477  break;
2478  case ISD::FMA:
2479  Res = WidenVecRes_Ternary(N);
2480  break;
2481  }
2482 
2483  // If Res is null, the sub-method took care of registering the result.
2484  if (Res.getNode())
2485  SetWidenedVector(SDValue(N, ResNo), Res);
2486 }
2487 
2488 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2489  // Ternary op widening.
2490  SDLoc dl(N);
2491  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2492  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2493  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2494  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2495  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2496 }
2497 
2498 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2499  // Binary op widening.
2500  SDLoc dl(N);
2501  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2502  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2503  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2504  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2505 }
2506 
2507 // Given a vector of operations that have been broken up to widen, see
2508 // if we can collect them together into the next widest legal VT. This
2509 // implementation is trap-safe.
2511  SmallVectorImpl<SDValue> &ConcatOps,
2512  unsigned ConcatEnd, EVT VT, EVT MaxVT,
2513  EVT WidenVT) {
2514  // Check to see if we have a single operation with the widen type.
2515  if (ConcatEnd == 1) {
2516  VT = ConcatOps[0].getValueType();
2517  if (VT == WidenVT)
2518  return ConcatOps[0];
2519  }
2520 
2521  SDLoc dl(ConcatOps[0]);
2522  EVT WidenEltVT = WidenVT.getVectorElementType();
2523  int Idx = 0;
2524 
2525  // while (Some element of ConcatOps is not of type MaxVT) {
2526  // From the end of ConcatOps, collect elements of the same type and put
2527  // them into an op of the next larger supported type
2528  // }
2529  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2530  Idx = ConcatEnd - 1;
2531  VT = ConcatOps[Idx--].getValueType();
2532  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2533  Idx--;
2534 
2535  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2536  EVT NextVT;
2537  do {
2538  NextSize *= 2;
2539  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2540  } while (!TLI.isTypeLegal(NextVT));
2541 
2542  if (!VT.isVector()) {
2543  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2544  SDValue VecOp = DAG.getUNDEF(NextVT);
2545  unsigned NumToInsert = ConcatEnd - Idx - 1;
2546  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2547  VecOp = DAG.getNode(
2548  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2549  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2550  }
2551  ConcatOps[Idx+1] = VecOp;
2552  ConcatEnd = Idx + 2;
2553  } else {
2554  // Vector type, create a CONCAT_VECTORS of type NextVT
2555  SDValue undefVec = DAG.getUNDEF(VT);
2556  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2557  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2558  unsigned RealVals = ConcatEnd - Idx - 1;
2559  unsigned SubConcatEnd = 0;
2560  unsigned SubConcatIdx = Idx + 1;
2561  while (SubConcatEnd < RealVals)
2562  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2563  while (SubConcatEnd < OpsToConcat)
2564  SubConcatOps[SubConcatEnd++] = undefVec;
2565  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2566  NextVT, SubConcatOps);
2567  ConcatEnd = SubConcatIdx + 1;
2568  }
2569  }
2570 
2571  // Check to see if we have a single operation with the widen type.
2572  if (ConcatEnd == 1) {
2573  VT = ConcatOps[0].getValueType();
2574  if (VT == WidenVT)
2575  return ConcatOps[0];
2576  }
2577 
2578  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2579  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2580  if (NumOps != ConcatEnd ) {
2581  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2582  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2583  ConcatOps[j] = UndefVal;
2584  }
2585  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2586  makeArrayRef(ConcatOps.data(), NumOps));
2587 }
2588 
2589 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2590  // Binary op widening for operations that can trap.
2591  unsigned Opcode = N->getOpcode();
2592  SDLoc dl(N);
2593  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2594  EVT WidenEltVT = WidenVT.getVectorElementType();
2595  EVT VT = WidenVT;
2596  unsigned NumElts = VT.getVectorNumElements();
2597  const SDNodeFlags Flags = N->getFlags();
2598  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2599  NumElts = NumElts / 2;
2600  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2601  }
2602 
2603  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2604  // Operation doesn't trap so just widen as normal.
2605  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2606  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2607  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2608  }
2609 
2610  // No legal vector version so unroll the vector operation and then widen.
2611  if (NumElts == 1)
2612  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2613 
2614  // Since the operation can trap, apply operation on the original vector.
2615  EVT MaxVT = VT;
2616  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2617  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2618  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2619 
2620  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2621  unsigned ConcatEnd = 0; // Current ConcatOps index.
2622  int Idx = 0; // Current Idx into input vectors.
2623 
2624  // NumElts := greatest legal vector size (at most WidenVT)
2625  // while (orig. vector has unhandled elements) {
2626  // take munches of size NumElts from the beginning and add to ConcatOps
2627  // NumElts := next smaller supported vector size or 1
2628  // }
2629  while (CurNumElts != 0) {
2630  while (CurNumElts >= NumElts) {
2631  SDValue EOp1 = DAG.getNode(
2632  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2633  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2634  SDValue EOp2 = DAG.getNode(
2635  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2636  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2637  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2638  Idx += NumElts;
2639  CurNumElts -= NumElts;
2640  }
2641  do {
2642  NumElts = NumElts / 2;
2643  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2644  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2645 
2646  if (NumElts == 1) {
2647  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2648  SDValue EOp1 = DAG.getNode(
2649  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2650  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2651  SDValue EOp2 = DAG.getNode(
2652  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2653  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2654  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2655  EOp1, EOp2, Flags);
2656  }
2657  CurNumElts = 0;
2658  }
2659  }
2660 
2661  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
2662 }
2663 
2664 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
2665  // StrictFP op widening for operations that can trap.
2666  unsigned NumOpers = N->getNumOperands();
2667  unsigned Opcode = N->getOpcode();
2668  SDLoc dl(N);
2669  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2670  EVT WidenEltVT = WidenVT.getVectorElementType();
2671  EVT VT = WidenVT;
2672  unsigned NumElts = VT.getVectorNumElements();
2673  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2674  NumElts = NumElts / 2;
2675  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2676  }
2677 
2678  // No legal vector version so unroll the vector operation and then widen.
2679  if (NumElts == 1)
2680  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2681 
2682  // Since the operation can trap, apply operation on the original vector.
2683  EVT MaxVT = VT;
2685  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2686 
2687  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2688  SmallVector<SDValue, 16> Chains;
2689  unsigned ConcatEnd = 0; // Current ConcatOps index.
2690  int Idx = 0; // Current Idx into input vectors.
2691 
2692  // The Chain is the first operand.
2693  InOps.push_back(N->getOperand(0));
2694 
2695  // Now process the remaining operands.
2696  for (unsigned i = 1; i < NumOpers; ++i) {
2697  SDValue Oper = N->getOperand(i);
2698 
2699  if (Oper.getValueType().isVector()) {
2700  assert(Oper.getValueType() == N->getValueType(0) &&
2701  "Invalid operand type to widen!");
2702  Oper = GetWidenedVector(Oper);
2703  }
2704 
2705  InOps.push_back(Oper);
2706  }
2707 
2708  // NumElts := greatest legal vector size (at most WidenVT)
2709  // while (orig. vector has unhandled elements) {
2710  // take munches of size NumElts from the beginning and add to ConcatOps
2711  // NumElts := next smaller supported vector size or 1
2712  // }
2713  while (CurNumElts != 0) {
2714  while (CurNumElts >= NumElts) {
2716 
2717  for (unsigned i = 0; i < NumOpers; ++i) {
2718  SDValue Op = InOps[i];
2719 
2720  if (Op.getValueType().isVector())
2721  Op = DAG.getNode(
2722  ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
2723  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2724 
2725  EOps.push_back(Op);
2726  }
2727 
2728  EVT OperVT[] = {VT, MVT::Other};
2729  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
2730  ConcatOps[ConcatEnd++] = Oper;
2731  Chains.push_back(Oper.getValue(1));
2732  Idx += NumElts;
2733  CurNumElts -= NumElts;
2734  }
2735  do {
2736  NumElts = NumElts / 2;
2737  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2738  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2739 
2740  if (NumElts == 1) {
2741  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2743 
2744  for (unsigned i = 0; i < NumOpers; ++i) {
2745  SDValue Op = InOps[i];
2746 
2747  if (Op.getValueType().isVector())
2748  Op = DAG.getNode(
2749  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
2750  DAG.getConstant(Idx, dl,
2751  TLI.getVectorIdxTy(DAG.getDataLayout())));
2752 
2753  EOps.push_back(Op);
2754  }
2755 
2756  EVT WidenVT[] = {WidenEltVT, MVT::Other};
2757  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
2758  ConcatOps[ConcatEnd++] = Oper;
2759  Chains.push_back(Oper.getValue(1));
2760  }
2761  CurNumElts = 0;
2762  }
2763  }
2764 
2765  // Build a factor node to remember all the Ops that have been created.
2766  SDValue NewChain;
2767  if (Chains.size() == 1)
2768  NewChain = Chains[0];
2769  else
2770  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
2771  ReplaceValueWith(SDValue(N, 1), NewChain);
2772 
2773  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
2774 }
2775 
2776 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
2777  SDValue InOp = N->getOperand(0);
2778  SDLoc DL(N);
2779 
2780  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2781  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2782 
2783  EVT InVT = InOp.getValueType();
2784  EVT InEltVT = InVT.getVectorElementType();
2785  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2786 
2787  unsigned Opcode = N->getOpcode();
2788  unsigned InVTNumElts = InVT.getVectorNumElements();
2789  const SDNodeFlags Flags = N->getFlags();
2790  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2791  InOp = GetWidenedVector(N->getOperand(0));
2792  InVT = InOp.getValueType();
2793  InVTNumElts = InVT.getVectorNumElements();
2794  if (InVTNumElts == WidenNumElts) {
2795  if (N->getNumOperands() == 1)
2796  return DAG.getNode(Opcode, DL, WidenVT, InOp);
2797  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
2798  }
2799  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
2800  // If both input and result vector types are of same width, extend
2801  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
2802  // accepts fewer elements in the result than in the input.
2803  if (Opcode == ISD::SIGN_EXTEND)
2804  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2805  if (Opcode == ISD::ZERO_EXTEND)
2806  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2807  }
2808  }
2809 
2810  if (TLI.isTypeLegal(InWidenVT)) {
2811  // Because the result and the input are different vector types, widening
2812  // the result could create a legal type but widening the input might make
2813  // it an illegal type that might lead to repeatedly splitting the input
2814  // and then widening it. To avoid this, we widen the input only if
2815  // it results in a legal type.
2816  if (WidenNumElts % InVTNumElts == 0) {
2817  // Widen the input and call convert on the widened input vector.
2818  unsigned NumConcat = WidenNumElts/InVTNumElts;
2819  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
2820  Ops[0] = InOp;
2821  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
2822  if (N->getNumOperands() == 1)
2823  return DAG.getNode(Opcode, DL, WidenVT, InVec);
2824  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
2825  }
2826 
2827  if (InVTNumElts % WidenNumElts == 0) {
2828  SDValue InVal = DAG.getNode(
2829  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
2830  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2831  // Extract the input and convert the shorten input vector.
2832  if (N->getNumOperands() == 1)
2833  return DAG.getNode(Opcode, DL, WidenVT, InVal);
2834  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
2835  }
2836  }
2837 
2838  // Otherwise unroll into some nasty scalar code and rebuild the vector.
2839  EVT EltVT = WidenVT.getVectorElementType();
2840  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
2841  // Use the original element count so we don't do more scalar opts than
2842  // necessary.
2843  unsigned MinElts = N->getValueType(0).getVectorNumElements();
2844  for (unsigned i=0; i < MinElts; ++i) {
2845  SDValue Val = DAG.getNode(
2846  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
2847  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2848  if (N->getNumOperands() == 1)
2849  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
2850  else
2851  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
2852  }
2853 
2854  return DAG.getBuildVector(WidenVT, DL, Ops);
2855 }
2856 
2857 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
2858  unsigned Opcode = N->getOpcode();
2859  SDValue InOp = N->getOperand(0);
2860  SDLoc DL(N);
2861 
2862  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2863  EVT WidenSVT = WidenVT.getVectorElementType();
2864  unsigned WidenNumElts = WidenVT.getVectorNumElements();
2865 
2866  EVT InVT = InOp.getValueType();
2867  EVT InSVT = InVT.getVectorElementType();
2868  unsigned InVTNumElts = InVT.getVectorNumElements();
2869 
2870  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2871  InOp = GetWidenedVector(InOp);
2872  InVT = InOp.getValueType();
2873  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
2874  switch (Opcode) {
2876  return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
2878  return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2880  return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2881  }
2882  }
2883  }
2884 
2885  // Unroll, extend the scalars and rebuild the vector.
2887  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
2888  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
2889  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2890  switch (Opcode) {
2892  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
2893  break;
2895  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
2896  break;
2898  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
2899  break;
2900  default:
2901  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
2902  }
2903  Ops.push_back(Val);
2904  }
2905 
2906  while (Ops.size() != WidenNumElts)
2907  Ops.push_back(DAG.getUNDEF(WidenSVT));
2908 
2909  return DAG.getBuildVector(WidenVT, DL, Ops);
2910 }
2911 
2912 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
2913  // If this is an FCOPYSIGN with same input types, we can treat it as a
2914  // normal (can trap) binary op.
2915  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
2916  return WidenVecRes_BinaryCanTrap(N);
2917 
2918  // If the types are different, fall back to unrolling.
2919  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2920  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2921 }
2922 
2923 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
2924  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2925  SDValue InOp = GetWidenedVector(N->getOperand(0));
2926  SDValue ShOp = N->getOperand(1);
2927  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2928 }
2929 
2930 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
2931  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2932  SDValue InOp = GetWidenedVector(N->getOperand(0));
2933  SDValue ShOp = N->getOperand(1);
2934 
2935  EVT ShVT = ShOp.getValueType();
2936  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
2937  ShOp = GetWidenedVector(ShOp);
2938  ShVT = ShOp.getValueType();
2939  }
2940  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
2941  ShVT.getVectorElementType(),
2942  WidenVT.getVectorNumElements());
2943  if (ShVT != ShWidenVT)
2944  ShOp = ModifyToType(ShOp, ShWidenVT);
2945 
2946  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2947 }
2948 
2949 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
2950  // Unary op widening.
2951  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2952  SDValue InOp = GetWidenedVector(N->getOperand(0));
2953  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
2954 }
2955 
2956 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
2957  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2958  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
2959  cast<VTSDNode>(N->getOperand(1))->getVT()
2960  .getVectorElementType(),
2961  WidenVT.getVectorNumElements());
2962  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
2963  return DAG.getNode(N->getOpcode(), SDLoc(N),
2964  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
2965 }
2966 
2967 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
2968  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2969  return GetWidenedVector(WidenVec);
2970 }
2971 
2972 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
2973  SDValue InOp = N->getOperand(0);
2974  EVT InVT = InOp.getValueType();
2975  EVT VT = N->getValueType(0);
2976  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2977  SDLoc dl(N);
2978 
2979  switch (getTypeAction(InVT)) {
2981  break;
2983  // If the incoming type is a vector that is being promoted, then
2984  // we know that the elements are arranged differently and that we
2985  // must perform the conversion using a stack slot.
2986  if (InVT.isVector())
2987  break;
2988 
2989  // If the InOp is promoted to the same size, convert it. Otherwise,
2990  // fall out of the switch and widen the promoted input.
2991  InOp = GetPromotedInteger(InOp);
2992  InVT = InOp.getValueType();
2993  if (WidenVT.bitsEq(InVT))
2994  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2995  break;
3002  break;
3004  // If the InOp is widened to the same size, convert it. Otherwise, fall
3005  // out of the switch and widen the widened input.
3006  InOp = GetWidenedVector(InOp);
3007  InVT = InOp.getValueType();
3008  if (WidenVT.bitsEq(InVT))
3009  // The input widens to the same size. Convert to the widen value.
3010  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3011  break;
3012  }
3013 
3014  unsigned WidenSize = WidenVT.getSizeInBits();
3015  unsigned InSize = InVT.getSizeInBits();
3016  // x86mmx is not an acceptable vector element type, so don't try.
3017  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3018  // Determine new input vector type. The new input vector type will use
3019  // the same element type (if its a vector) or use the input type as a
3020  // vector. It is the same size as the type to widen to.
3021  EVT NewInVT;
3022  unsigned NewNumElts = WidenSize / InSize;
3023  if (InVT.isVector()) {
3024  EVT InEltVT = InVT.getVectorElementType();
3025  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3026  WidenSize / InEltVT.getSizeInBits());
3027  } else {
3028  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3029  }
3030 
3031  if (TLI.isTypeLegal(NewInVT)) {
3032  SDValue NewVec;
3033  if (InVT.isVector()) {
3034  // Because the result and the input are different vector types, widening
3035  // the result could create a legal type but widening the input might make
3036  // it an illegal type that might lead to repeatedly splitting the input
3037  // and then widening it. To avoid this, we widen the input only if
3038  // it results in a legal type.
3039  SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3040  Ops[0] = InOp;
3041 
3042  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3043  } else {
3044  NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3045  }
3046  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3047  }
3048  }
3049 
3050  return CreateStackStoreLoad(InOp, WidenVT);
3051 }
3052 
3053 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3054  SDLoc dl(N);
3055  // Build a vector with undefined for the new nodes.
3056  EVT VT = N->getValueType(0);
3057 
3058  // Integer BUILD_VECTOR operands may be larger than the node's vector element
3059  // type. The UNDEFs need to have the same type as the existing operands.
3060  EVT EltVT = N->getOperand(0).getValueType();
3061  unsigned NumElts = VT.getVectorNumElements();
3062 
3063  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3064  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3065 
3066  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3067  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3068  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3069 
3070  return DAG.getBuildVector(WidenVT, dl, NewOps);
3071 }
3072 
3073 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3074  EVT InVT = N->getOperand(0).getValueType();
3075  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3076  SDLoc dl(N);
3077  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3078  unsigned NumInElts = InVT.getVectorNumElements();
3079  unsigned NumOperands = N->getNumOperands();
3080 
3081  bool InputWidened = false; // Indicates we need to widen the input.
3082  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3083  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
3084  // Add undef vectors to widen to correct length.
3085  unsigned NumConcat = WidenVT.getVectorNumElements() /
3086  InVT.getVectorNumElements();
3087  SDValue UndefVal = DAG.getUNDEF(InVT);
3088  SmallVector<SDValue, 16> Ops(NumConcat);
3089  for (unsigned i=0; i < NumOperands; ++i)
3090  Ops[i] = N->getOperand(i);
3091  for (unsigned i = NumOperands; i != NumConcat; ++i)
3092  Ops[i] = UndefVal;
3093  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3094  }
3095  } else {
3096  InputWidened = true;
3097  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3098  // The inputs and the result are widen to the same value.
3099  unsigned i;
3100  for (i=1; i < NumOperands; ++i)
3101  if (!N->getOperand(i).isUndef())
3102  break;
3103 
3104  if (i == NumOperands)
3105  // Everything but the first operand is an UNDEF so just return the
3106  // widened first operand.
3107  return GetWidenedVector(N->getOperand(0));
3108 
3109  if (NumOperands == 2) {
3110  // Replace concat of two operands with a shuffle.
3111  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3112  for (unsigned i = 0; i < NumInElts; ++i) {
3113  MaskOps[i] = i;
3114  MaskOps[i + NumInElts] = i + WidenNumElts;
3115  }
3116  return DAG.getVectorShuffle(WidenVT, dl,
3117  GetWidenedVector(N->getOperand(0)),
3118  GetWidenedVector(N->getOperand(1)),
3119  MaskOps);
3120  }
3121  }
3122  }
3123 
3124  // Fall back to use extracts and build vector.
3125  EVT EltVT = WidenVT.getVectorElementType();
3126  SmallVector<SDValue, 16> Ops(WidenNumElts);
3127  unsigned Idx = 0;
3128  for (unsigned i=0; i < NumOperands; ++i) {
3129  SDValue InOp = N->getOperand(i);
3130  if (InputWidened)
3131  InOp = GetWidenedVector(InOp);
3132  for (unsigned j=0; j < NumInElts; ++j)
3133  Ops[Idx++] = DAG.getNode(
3134  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3135  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3136  }
3137  SDValue UndefVal = DAG.getUNDEF(EltVT);
3138  for (; Idx < WidenNumElts; ++Idx)
3139  Ops[Idx] = UndefVal;
3140  return DAG.getBuildVector(WidenVT, dl, Ops);
3141 }
3142 
3143 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3144  EVT VT = N->getValueType(0);
3145  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3146  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3147  SDValue InOp = N->getOperand(0);
3148  SDValue Idx = N->getOperand(1);
3149  SDLoc dl(N);
3150 
3151  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3152  InOp = GetWidenedVector(InOp);
3153 
3154  EVT InVT = InOp.getValueType();
3155 
3156  // Check if we can just return the input vector after widening.
3157  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3158  if (IdxVal == 0 && InVT == WidenVT)
3159  return InOp;
3160 
3161  // Check if we can extract from the vector.
3162  unsigned InNumElts = InVT.getVectorNumElements();
3163  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3164  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3165 
3166  // We could try widening the input to the right length but for now, extract
3167  // the original elements, fill the rest with undefs and build a vector.
3168  SmallVector<SDValue, 16> Ops(WidenNumElts);
3169  EVT EltVT = VT.getVectorElementType();
3170  unsigned NumElts = VT.getVectorNumElements();
3171  unsigned i;
3172  for (i=0; i < NumElts; ++i)
3173  Ops[i] =
3174  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3175  DAG.getConstant(IdxVal + i, dl,
3176  TLI.getVectorIdxTy(DAG.getDataLayout())));
3177 
3178  SDValue UndefVal = DAG.getUNDEF(EltVT);
3179  for (; i < WidenNumElts; ++i)
3180  Ops[i] = UndefVal;
3181  return DAG.getBuildVector(WidenVT, dl, Ops);
3182 }
3183 
3184 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3185  SDValue InOp = GetWidenedVector(N->getOperand(0));
3186  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3187  InOp.getValueType(), InOp,
3188  N->getOperand(1), N->getOperand(2));
3189 }
3190 
3191 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3192  LoadSDNode *LD = cast<LoadSDNode>(N);
3194 
3195  SDValue Result;
3196  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3197  if (ExtType != ISD::NON_EXTLOAD)
3198  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3199  else
3200  Result = GenWidenVectorLoads(LdChain, LD);
3201 
3202  // If we generate a single load, we can use that for the chain. Otherwise,
3203  // build a factor node to remember the multiple loads are independent and
3204  // chain to that.
3205  SDValue NewChain;
3206  if (LdChain.size() == 1)
3207  NewChain = LdChain[0];
3208  else
3209  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3210 
3211  // Modified the chain - switch anything that used the old chain to use
3212  // the new one.
3213  ReplaceValueWith(SDValue(N, 1), NewChain);
3214 
3215  return Result;
3216 }
3217 
3218 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3219 
3220  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3221  SDValue Mask = N->getMask();
3222  EVT MaskVT = Mask.getValueType();
3223  SDValue PassThru = GetWidenedVector(N->getPassThru());
3225  SDLoc dl(N);
3226 
3227  // The mask should be widened as well
3228  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3229  MaskVT.getVectorElementType(),
3230  WidenVT.getVectorNumElements());
3231  Mask = ModifyToType(Mask, WideMaskVT, true);
3232 
3233  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3234  Mask, PassThru, N->getMemoryVT(),
3235  N->getMemOperand(), ExtType,
3236  N->isExpandingLoad());
3237  // Legalize the chain result - switch anything that used the old chain to
3238  // use the new one.
3239  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3240  return Res;
3241 }
3242 
3243 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3244 
3245  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3246  SDValue Mask = N->getMask();
3247  EVT MaskVT = Mask.getValueType();
3248  SDValue PassThru = GetWidenedVector(N->getPassThru());
3249  SDValue Scale = N->getScale();
3250  unsigned NumElts = WideVT.getVectorNumElements();
3251  SDLoc dl(N);
3252 
3253  // The mask should be widened as well
3254  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3255  MaskVT.getVectorElementType(),
3256  WideVT.getVectorNumElements());
3257  Mask = ModifyToType(Mask, WideMaskVT, true);
3258 
3259  // Widen the Index operand
3260  SDValue Index = N->getIndex();
3261  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3262  Index.getValueType().getScalarType(),
3263  NumElts);
3264  Index = ModifyToType(Index, WideIndexVT);
3265  SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3266  Scale };
3267  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3268  N->getMemoryVT(), dl, Ops,
3269  N->getMemOperand());
3270 
3271  // Legalize the chain result - switch anything that used the old chain to
3272  // use the new one.
3273  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3274  return Res;
3275 }
3276 
3277 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3278  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3279  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3280  WidenVT, N->getOperand(0));
3281 }
3282 
3283 // Return true if this is a node that could have two SETCCs as operands.
3284 static inline bool isLogicalMaskOp(unsigned Opcode) {
3285  switch (Opcode) {
3286  case ISD::AND:
3287  case ISD::OR:
3288  case ISD::XOR:
3289  return true;
3290  }
3291  return false;
3292 }
3293 
3294 // This is used just for the assert in convertMask(). Check that this either
3295 // a SETCC or a previously handled SETCC by convertMask().
3296 #ifndef NDEBUG
3297 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3298  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3299  N = N.getOperand(0);
3300  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3301  for (unsigned i = 1; i < N->getNumOperands(); ++i)
3302  if (!N->getOperand(i)->isUndef())
3303  return false;
3304  N = N.getOperand(0);
3305  }
3306 
3307  if (N.getOpcode() == ISD::TRUNCATE)
3308  N = N.getOperand(0);
3309  else if (N.getOpcode() == ISD::SIGN_EXTEND)
3310  N = N.getOperand(0);
3311 
3312  if (isLogicalMaskOp(N.getOpcode()))
3313  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3315 
3316  return (N.getOpcode() == ISD::SETCC ||
3318 }
3319 #endif
3320 
3321 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3322 // to ToMaskVT if needed with vector extension or truncation.
3323 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3324  EVT ToMaskVT) {
3325  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3326  // FIXME: This code seems to be too restrictive, we might consider
3327  // generalizing it or dropping it.
3328  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3329 
3330  // Make a new Mask node, with a legal result VT.
3332  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3333  Ops.push_back(InMask->getOperand(i));
3334  SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3335 
3336  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3337  // extend or truncate is needed.
3338  LLVMContext &Ctx = *DAG.getContext();
3339  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3340  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3341  if (MaskScalarBits < ToMaskScalBits) {
3342  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3343  MaskVT.getVectorNumElements());
3344  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3345  } else if (MaskScalarBits > ToMaskScalBits) {
3346  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3347  MaskVT.getVectorNumElements());
3348  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3349  }
3350 
3352  ToMaskVT.getScalarSizeInBits() &&
3353  "Mask should have the right element size by now.");
3354 
3355  // Adjust Mask to the right number of elements.
3356  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3357  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3358  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3359  SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3360  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3361  ZeroIdx);
3362  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3363  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3364  EVT SubVT = Mask->getValueType(0);
3365  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3366  SubOps[0] = Mask;
3367  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3368  }
3369 
3370  assert((Mask->getValueType(0) == ToMaskVT) &&
3371  "A mask of ToMaskVT should have been produced by now.");
3372 
3373  return Mask;
3374 }
3375 
3376 // Get the target mask VT, and widen if needed.
3377 EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
3378  assert(SetCC->getOpcode() == ISD::SETCC);
3379  LLVMContext &Ctx = *DAG.getContext();
3380  EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
3381  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3382  MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
3383  return MaskVT;
3384 }
3385 
3386 // This method tries to handle VSELECT and its mask by legalizing operands
3387 // (which may require widening) and if needed adjusting the mask vector type
3388 // to match that of the VSELECT. Without it, many cases end up with
3389 // scalarization of the SETCC, with many unnecessary instructions.
3390 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3391  LLVMContext &Ctx = *DAG.getContext();
3392  SDValue Cond = N->getOperand(0);
3393 
3394  if (N->getOpcode() != ISD::VSELECT)
3395  return SDValue();
3396 
3397  if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3398  return SDValue();
3399 
3400  // If this is a splitted VSELECT that was previously already handled, do
3401  // nothing.
3402  EVT CondVT = Cond->getValueType(0);
3403  if (CondVT.getScalarSizeInBits() != 1)
3404  return SDValue();
3405 
3406  EVT VSelVT = N->getValueType(0);
3407  // Only handle vector types which are a power of 2.
3408  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3409  return SDValue();
3410 
3411  // Don't touch if this will be scalarized.
3412  EVT FinalVT = VSelVT;
3413  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3414  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3415 
3416  if (FinalVT.getVectorNumElements() == 1)
3417  return SDValue();
3418 
3419  // If there is support for an i1 vector mask, don't touch.
3420  if (Cond.getOpcode() == ISD::SETCC) {
3421  EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3422  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3423  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3424  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3425  if (SetCCResVT.getScalarSizeInBits() == 1)
3426  return SDValue();
3427  } else if (CondVT.getScalarType() == MVT::i1) {
3428  // If there is support for an i1 vector mask (or only scalar i1 conditions),
3429  // don't touch.
3430  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3431  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3432 
3433  if (CondVT.getScalarType() == MVT::i1)
3434  return SDValue();
3435  }
3436 
3437  // Get the VT and operands for VSELECT, and widen if needed.
3438  SDValue VSelOp1 = N->getOperand(1);
3439  SDValue VSelOp2 = N->getOperand(2);
3440  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3441  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3442  VSelOp1 = GetWidenedVector(VSelOp1);
3443  VSelOp2 = GetWidenedVector(VSelOp2);
3444  }
3445 
3446  // The mask of the VSELECT should have integer elements.
3447  EVT ToMaskVT = VSelVT;
3448  if (!ToMaskVT.getScalarType().isInteger())
3449  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3450 
3451  SDValue Mask;
3452  if (Cond->getOpcode() == ISD::SETCC) {
3453  EVT MaskVT = getSETCCWidenedResultTy(Cond);
3454  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3455  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3456  Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3457  Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3458  // Cond is (AND/OR/XOR (SETCC, SETCC))
3459  SDValue SETCC0 = Cond->getOperand(0);
3460  SDValue SETCC1 = Cond->getOperand(1);
3461  EVT VT0 = getSETCCWidenedResultTy(SETCC0);
3462  EVT VT1 = getSETCCWidenedResultTy(SETCC1);
3463  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3464  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3465  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3466  EVT MaskVT;
3467  // If the two SETCCs have different VTs, either extend/truncate one of
3468  // them to the other "towards" ToMaskVT, or truncate one and extend the
3469  // other to ToMaskVT.
3470  if (ScalarBits0 != ScalarBits1) {
3471  EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3472  EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3473  if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3474  MaskVT = WideVT;
3475  else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3476  MaskVT = NarrowVT;
3477  else
3478  MaskVT = ToMaskVT;
3479  } else
3480  // If the two SETCCs have the same VT, don't change it.
3481  MaskVT = VT0;
3482 
3483  // Make new SETCCs and logical nodes.
3484  SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3485  SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3486  Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3487 
3488  // Convert the logical op for VSELECT if needed.
3489  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3490  } else
3491  return SDValue();
3492 
3493  return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3494 }
3495 
3496 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3497  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3498  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3499 
3500  SDValue Cond1 = N->getOperand(0);
3501  EVT CondVT = Cond1.getValueType();
3502  if (CondVT.isVector()) {
3503  if (SDValue Res = WidenVSELECTAndMask(N))
3504  return Res;
3505 
3506  EVT CondEltVT = CondVT.getVectorElementType();
3507  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3508  CondEltVT, WidenNumElts);
3509  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3510  Cond1 = GetWidenedVector(Cond1);
3511 
3512  // If we have to split the condition there is no point in widening the
3513  // select. This would result in an cycle of widening the select ->
3514  // widening the condition operand -> splitting the condition operand ->
3515  // splitting the select -> widening the select. Instead split this select
3516  // further and widen the resulting type.
3517  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3518  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3519  SDValue Res = ModifyToType(SplitSelect, WidenVT);
3520  return Res;
3521  }
3522 
3523  if (Cond1.getValueType() != CondWidenVT)
3524  Cond1 = ModifyToType(Cond1, CondWidenVT);
3525  }
3526 
3527  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3528  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3529  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3530  return DAG.getNode(N->getOpcode(), SDLoc(N),
3531  WidenVT, Cond1, InOp1, InOp2);
3532 }
3533 
3534 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3535  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3536  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3537  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3538  InOp1.getValueType(), N->getOperand(0),
3539  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3540 }
3541 
3542 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3543  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3544  return DAG.getUNDEF(WidenVT);
3545 }
3546 
3547 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3548  EVT VT = N->getValueType(0);
3549  SDLoc dl(N);
3550 
3551  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3552  unsigned NumElts = VT.getVectorNumElements();
3553  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3554 
3555  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3556  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3557 
3558  // Adjust mask based on new input vector length.
3559  SmallVector<int, 16> NewMask;
3560  for (unsigned i = 0; i != NumElts; ++i) {
3561  int Idx = N->getMaskElt(i);
3562  if (Idx < (int)NumElts)
3563  NewMask.push_back(Idx);
3564  else
3565  NewMask.push_back(Idx - NumElts + WidenNumElts);
3566  }
3567  for (unsigned i = NumElts; i != WidenNumElts; ++i)
3568  NewMask.push_back(-1);
3569  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
3570 }
3571 
3572 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
3573  assert(N->getValueType(0).isVector() &&
3574  N->getOperand(0).getValueType().isVector() &&
3575  "Operands must be vectors");
3576  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3577  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3578 
3579  SDValue InOp1 = N->getOperand(0);
3580  EVT InVT = InOp1.getValueType();
3581  assert(InVT.isVector() && "can not widen non-vector type");
3582  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
3583  InVT.getVectorElementType(), WidenNumElts);
3584 
3585  // The input and output types often differ here, and it could be that while
3586  // we'd prefer to widen the result type, the input operands have been split.
3587  // In this case, we also need to split the result of this node as well.
3588  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
3589  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
3590  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
3591  return Res;
3592  }
3593 
3594  InOp1 = GetWidenedVector(InOp1);
3595  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3596 
3597  // Assume that the input and output will be widen appropriately. If not,
3598  // we will have to unroll it at some point.
3599  assert(InOp1.getValueType() == WidenInVT &&
3600  InOp2.getValueType() == WidenInVT &&
3601  "Input not widened to expected type!");
3602  (void)WidenInVT;
3603  return DAG.getNode(ISD::SETCC, SDLoc(N),
3604  WidenVT, InOp1, InOp2, N->getOperand(2));
3605 }
3606 
3607 
3608 //===----------------------------------------------------------------------===//
3609 // Widen Vector Operand
3610 //===----------------------------------------------------------------------===//
3611 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
3612  LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
3613  dbgs() << "\n");
3614  SDValue Res = SDValue();
3615 
3616  // See if the target wants to custom widen this node.
3617  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3618  return false;
3619 
3620  switch (N->getOpcode()) {
3621  default:
3622 #ifndef NDEBUG
3623  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
3624  N->dump(&DAG);
3625  dbgs() << "\n";
3626 #endif
3627  llvm_unreachable("Do not know how to widen this operator's operand!");
3628 
3629  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
3630  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
3631  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
3632  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
3633  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
3634  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
3635  case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
3636  case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
3637  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
3638  case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
3639 
3640  case ISD::ANY_EXTEND:
3641  case ISD::SIGN_EXTEND:
3642  case ISD::ZERO_EXTEND:
3643  Res = WidenVecOp_EXTEND(N);
3644  break;
3645 
3646  case ISD::FP_EXTEND:
3647  case ISD::FP_TO_SINT:
3648  case ISD::FP_TO_UINT:
3649  case ISD::SINT_TO_FP:
3650  case ISD::UINT_TO_FP:
3651  case ISD::TRUNCATE:
3652  Res = WidenVecOp_Convert(N);
3653  break;
3654  }
3655 
3656  // If Res is null, the sub-method took care of registering the result.
3657  if (!Res.getNode()) return false;
3658 
3659  // If the result is N, the sub-method updated N in place. Tell the legalizer
3660  // core about this.
3661  if (Res.getNode() == N)
3662  return true;
3663 
3664 
3665  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3666  "Invalid operand expansion");
3667 
3668  ReplaceValueWith(SDValue(N, 0), Res);
3669  return false;
3670 }
3671 
3672 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
3673  SDLoc DL(N);
3674  EVT VT = N->getValueType(0);
3675 
3676  SDValue InOp = N->getOperand(0);
3677  assert(getTypeAction(InOp.getValueType()) ==
3679  "Unexpected type action");
3680  InOp = GetWidenedVector(InOp);
3683  "Input wasn't widened!");
3684 
3685  // We may need to further widen the operand until it has the same total
3686  // vector size as the result.
3687  EVT InVT = InOp.getValueType();
3688  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
3689  EVT InEltVT = InVT.getVectorElementType();
3690  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
3691  EVT FixedVT = (MVT::SimpleValueType)i;
3692  EVT FixedEltVT = FixedVT.getVectorElementType();
3693  if (TLI.isTypeLegal(FixedVT) &&
3694  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
3695  FixedEltVT == InEltVT) {
3696  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
3697  "Not enough elements in the fixed type for the operand!");
3698  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
3699  "We can't have the same type as we started with!");
3700  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
3701  InOp = DAG.getNode(
3702  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
3703  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3704  else
3705  InOp = DAG.getNode(
3706  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
3707  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3708  break;
3709  }
3710  }
3711  InVT = InOp.getValueType();
3712  if (InVT.getSizeInBits() != VT.getSizeInBits())
3713  // We couldn't find a legal vector type that was a widening of the input
3714  // and could be extended in-register to the result type, so we have to
3715  // scalarize.
3716  return WidenVecOp_Convert(N);
3717  }
3718 
3719  // Use special DAG nodes to represent the operation of extending the
3720  // low lanes.
3721  switch (N->getOpcode()) {
3722  default:
3723  llvm_unreachable("Extend legalization on extend operation!");
3724  case ISD::ANY_EXTEND:
3725  return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
3726  case ISD::SIGN_EXTEND:
3727  return DAG.getSignExtendVectorInReg(InOp, DL, VT);
3728  case ISD::ZERO_EXTEND:
3729  return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
3730  }
3731 }
3732 
3733 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
3734  // The result (and first input) is legal, but the second input is illegal.
3735  // We can't do much to fix that, so just unroll and let the extracts off of
3736  // the second input be widened as needed later.
3737  return DAG.UnrollVectorOp(N);
3738 }
3739 
3740 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
3741  // Since the result is legal and the input is illegal.
3742  EVT VT = N->getValueType(0);
3743  EVT EltVT = VT.getVectorElementType();
3744  SDLoc dl(N);
3745  unsigned NumElts = VT.getVectorNumElements();
3746  SDValue InOp = N->getOperand(0);
3747  assert(getTypeAction(InOp.getValueType()) ==
3749  "Unexpected type action");
3750  InOp = GetWidenedVector(InOp);
3751  EVT InVT = InOp.getValueType();
3752  unsigned Opcode = N->getOpcode();
3753 
3754  // See if a widened result type would be legal, if so widen the node.
3755  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3756  InVT.getVectorNumElements());
3757  if (TLI.isTypeLegal(WideVT)) {
3758  SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp);
3759  return DAG.getNode(
3760  ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
3761  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3762  }
3763 
3764  EVT InEltVT = InVT.getVectorElementType();
3765 
3766  // Unroll the convert into some scalar code and create a nasty build vector.
3767  SmallVector<SDValue, 16> Ops(NumElts);
3768  for (unsigned i=0; i < NumElts; ++i)
3769  Ops[i] = DAG.getNode(
3770  Opcode, dl, EltVT,
3771  DAG.getNode(
3772  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
3773  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3774 
3775  return DAG.getBuildVector(VT, dl, Ops);
3776 }
3777 
3778 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
3779  EVT VT = N->getValueType(0);
3780  SDValue InOp = GetWidenedVector(N->getOperand(0));
3781  EVT InWidenVT = InOp.getValueType();
3782  SDLoc dl(N);
3783 
3784  // Check if we can convert between two legal vector types and extract.
3785  unsigned InWidenSize = InWidenVT.getSizeInBits();
3786  unsigned Size = VT.getSizeInBits();
3787  // x86mmx is not an acceptable vector element type, so don't try.
3788  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
3789  unsigned NewNumElts = InWidenSize / Size;
3790  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
3791  if (TLI.isTypeLegal(NewVT)) {
3792  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
3793  return DAG.getNode(
3794  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
3795  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3796  }
3797  }
3798 
3799  return CreateStackStoreLoad(InOp, VT);
3800 }
3801 
3802 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
3803  EVT VT = N->getValueType(0);
3804  EVT EltVT = VT.getVectorElementType();
3805  EVT InVT = N->getOperand(0).getValueType();
3806  SDLoc dl(N);
3807 
3808  // If the widen width for this operand is the same as the width of the concat
3809  // and all but the first operand is undef, just use the widened operand.
3810  unsigned NumOperands = N->getNumOperands();
3811  if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3812  unsigned i;
3813  for (i = 1; i < NumOperands; ++i)
3814  if (!N->getOperand(i).isUndef())
3815  break;
3816 
3817  if (i == NumOperands)
3818  return GetWidenedVector(N->getOperand(0));
3819  }
3820 
3821  // Otherwise, fall back to a nasty build vector.
3822  unsigned NumElts = VT.getVectorNumElements();
3823  SmallVector<SDValue, 16> Ops(NumElts);
3824 
3825  unsigned NumInElts = InVT.getVectorNumElements();
3826 
3827  unsigned Idx = 0;
3828  for (unsigned i=0; i < NumOperands; ++i) {
3829  SDValue InOp = N->getOperand(i);
3830  assert(getTypeAction(InOp.getValueType()) ==
3832  "Unexpected type action");
3833  InOp = GetWidenedVector(InOp);
3834  for (unsigned j=0; j < NumInElts; ++j)
3835  Ops[Idx++] = DAG.getNode(
3836  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3837  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3838  }
3839  return DAG.getBuildVector(VT, dl, Ops);
3840 }
3841 
3842 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3843  SDValue InOp = GetWidenedVector(N->getOperand(0));
3844  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
3845  N->getValueType(0), InOp, N->getOperand(1));
3846 }
3847 
3848 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3849  SDValue InOp = GetWidenedVector(N->getOperand(0));
3850  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3851  N->getValueType(0), InOp, N->getOperand(1));
3852 }
3853 
3854 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
3855  // We have to widen the value, but we want only to store the original
3856  // vector type.
3857  StoreSDNode *ST = cast<StoreSDNode>(N);
3858 
3859  if (!ST->getMemoryVT().getScalarType().isByteSized())
3860  return TLI.scalarizeVectorStore(ST, DAG);
3861 
3862  SmallVector<SDValue, 16> StChain;
3863  if (ST->isTruncatingStore())
3864  GenWidenVectorTruncStores(StChain, ST);
3865  else
3866  GenWidenVectorStores(StChain, ST);
3867 
3868  if (StChain.size() == 1)
3869  return StChain[0];
3870  else
3871  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
3872 }
3873 
3874 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
3875  assert((OpNo == 1 || OpNo == 3) &&
3876  "Can widen only data or mask operand of mstore");
3877  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
3878  SDValue Mask = MST->getMask();
3879  EVT MaskVT = Mask.getValueType();
3880  SDValue StVal = MST->getValue();
3881  SDLoc dl(N);
3882 
3883  if (OpNo == 1) {
3884  // Widen the value.
3885  StVal = GetWidenedVector(StVal);
3886 
3887  // The mask should be widened as well.
3888  EVT WideVT = StVal.getValueType();
3889  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3890  MaskVT.getVectorElementType(),
3891  WideVT.getVectorNumElements());
3892  Mask = ModifyToType(Mask, WideMaskVT, true);
3893  } else {
3894  // Widen the mask.
3895  EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
3896  Mask = ModifyToType(Mask, WideMaskVT, true);
3897 
3898  EVT ValueVT = StVal.getValueType();
3899  EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
3900  ValueVT.getVectorElementType(),
3901  WideMaskVT.getVectorNumElements());
3902  StVal = ModifyToType(StVal, WideVT);
3903  }
3904 
3905  assert(Mask.getValueType().getVectorNumElements() ==
3906  StVal.getValueType().getVectorNumElements() &&
3907  "Mask and data vectors should have the same number of elements");
3908  return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
3909  Mask, MST->getMemoryVT(), MST->getMemOperand(),
3910  false, MST->isCompressingStore());
3911 }
3912 
3913 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
3914  assert(OpNo == 4 && "Can widen only the index of mgather");
3915  auto *MG = cast<MaskedGatherSDNode>(N);
3916  SDValue DataOp = MG->getPassThru();
3917  SDValue Mask = MG->getMask();
3918  SDValue Scale = MG->getScale();
3919 
3920  // Just widen the index. It's allowed to have extra elements.
3921  SDValue Index = GetWidenedVector(MG->getIndex());
3922 
3923  SDLoc dl(N);
3924  SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
3925  Scale};
3926  SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
3927  MG->getMemOperand());
3928  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3929  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
3930  return SDValue();
3931 }
3932 
3933 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
3934  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
3935  SDValue DataOp = MSC->getValue();
3936  SDValue Mask = MSC->getMask();
3937  SDValue Index = MSC->getIndex();
3938  SDValue Scale = MSC->getScale();
3939 
3940  unsigned NumElts;
3941  if (OpNo == 1) {
3942  DataOp = GetWidenedVector(DataOp);
3943  NumElts = DataOp.getValueType().getVectorNumElements();
3944 
3945  // Widen index.
3946  EVT IndexVT = Index.getValueType();
3947  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3948  IndexVT.getVectorElementType(), NumElts);
3949  Index = ModifyToType(Index, WideIndexVT);
3950 
3951  // The mask should be widened as well.
3952  EVT MaskVT = Mask.getValueType();
3953  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3954  MaskVT.getVectorElementType(), NumElts);
3955  Mask = ModifyToType(Mask, WideMaskVT, true);
3956  } else if (OpNo == 4) {
3957  // Just widen the index. It's allowed to have extra elements.
3958  Index = GetWidenedVector(Index);
3959  } else
3960  llvm_unreachable("Can't widen this operand of mscatter");
3961 
3962  SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
3963  Scale};
3964  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
3965  MSC->getMemoryVT(), SDLoc(N), Ops,
3966  MSC->getMemOperand());
3967 }
3968 
3969 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
3970  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
3971  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3972  SDLoc dl(N);
3973  EVT VT = N->getValueType(0);
3974 
3975  // WARNING: In this code we widen the compare instruction with garbage.
3976  // This garbage may contain denormal floats which may be slow. Is this a real
3977  // concern ? Should we zero the unused lanes if this is a float compare ?
3978 
3979  // Get a new SETCC node to compare the newly widened operands.
3980  // Only some of the compared elements are legal.
3981  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3982  InOp0.getValueType());
3983  // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
3984  if (VT.getScalarType() == MVT::i1)
3985  SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
3986  SVT.getVectorNumElements());
3987 
3988  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
3989  SVT, InOp0, InOp1, N->getOperand(2));
3990 
3991  // Extract the needed results from the result vector.
3992  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
3993  SVT.getVectorElementType(),
3994  VT.getVectorNumElements());
3995  SDValue CC = DAG.getNode(
3996  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
3997  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3998 
3999  return PromoteTargetBoolean(CC, VT);
4000 }
4001 
4002 
4003 //===----------------------------------------------------------------------===//
4004 // Vector Widening Utilities
4005 //===----------------------------------------------------------------------===//
4006 
4007 // Utility function to find the type to chop up a widen vector for load/store
4008 // TLI: Target lowering used to determine legal types.
4009 // Width: Width left need to load/store.
4010 // WidenVT: The widen vector type to load to/store from
4011 // Align: If 0, don't allow use of a wider type
4012 // WidenEx: If Align is not 0, the amount additional we can load/store from.
4013 
4014 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4015  unsigned Width, EVT WidenVT,
4016  unsigned Align = 0, unsigned WidenEx = 0) {
4017  EVT WidenEltVT = WidenVT.getVectorElementType();
4018  unsigned WidenWidth = WidenVT.getSizeInBits();
4019  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4020  unsigned AlignInBits = Align*8;
4021 
4022  // If we have one element to load/store, return it.
4023  EVT RetVT = WidenEltVT;
4024  if (Width == WidenEltWidth)
4025  return RetVT;
4026 
4027  // See if there is larger legal integer than the element type to load/store.
4028  unsigned VT;
4029  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4030  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
4031  EVT MemVT((MVT::SimpleValueType) VT);
4032  unsigned MemVTWidth = MemVT.getSizeInBits();
4033  if (MemVT.getSizeInBits() <= WidenEltWidth)
4034  break;
4035  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4036  if ((Action == TargetLowering::TypeLegal ||
4038  (WidenWidth % MemVTWidth) == 0 &&
4039  isPowerOf2_32(WidenWidth / MemVTWidth) &&
4040  (MemVTWidth <= Width ||
4041  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4042  RetVT = MemVT;
4043  break;
4044  }
4045  }
4046 
4047  // See if there is a larger vector type to load/store that has the same vector
4048  // element type and is evenly divisible with the WidenVT.
4049  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4050  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
4051  EVT MemVT = (MVT::SimpleValueType) VT;
4052  unsigned MemVTWidth = MemVT.getSizeInBits();
4053  if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
4054  (WidenWidth % MemVTWidth) == 0 &&
4055  isPowerOf2_32(WidenWidth / MemVTWidth) &&
4056  (MemVTWidth <= Width ||
4057  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4058  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
4059  return MemVT;
4060  }
4061  }
4062 
4063  return RetVT;
4064 }
4065 
4066 // Builds a vector type from scalar loads
4067 // VecTy: Resulting Vector type
4068 // LDOps: Load operators to build a vector type
4069 // [Start,End) the list of loads to use.
4071  SmallVectorImpl<SDValue> &LdOps,
4072  unsigned Start, unsigned End) {
4073  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4074  SDLoc dl(LdOps[Start]);
4075  EVT LdTy = LdOps[Start].getValueType();
4076  unsigned Width = VecTy.getSizeInBits();
4077  unsigned NumElts = Width / LdTy.getSizeInBits();
4078  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4079 
4080  unsigned Idx = 1;
4081  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4082 
4083  for (unsigned i = Start + 1; i != End; ++i) {
4084  EVT NewLdTy = LdOps[i].getValueType();
4085  if (NewLdTy != LdTy) {
4086  NumElts = Width / NewLdTy.getSizeInBits();
4087  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4088  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4089  // Readjust position and vector position based on new load type.
4090  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4091  LdTy = NewLdTy;
4092  }
4093  VecOp = DAG.getNode(
4094  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4095  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4096  }
4097  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4098 }
4099 
4100 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4101  LoadSDNode *LD) {
4102  // The strategy assumes that we can efficiently load power-of-two widths.
4103  // The routine chops the vector into the largest vector loads with the same
4104  // element type or scalar loads and then recombines it to the widen vector
4105  // type.
4106  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4107  unsigned WidenWidth = WidenVT.getSizeInBits();
4108  EVT LdVT = LD->getMemoryVT();
4109  SDLoc dl(LD);
4110  assert(LdVT.isVector() && WidenVT.isVector());
4111  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4112 
4113  // Load information
4114  SDValue Chain = LD->getChain();
4115  SDValue BasePtr = LD->getBasePtr();
4116  unsigned Align = LD->getAlignment();
4117  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4118  AAMDNodes AAInfo = LD->getAAInfo();
4119 
4120  int LdWidth = LdVT.getSizeInBits();
4121  int WidthDiff = WidenWidth - LdWidth;
4122  unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
4123 
4124  // Find the vector type that can load from.
4125  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4126  int NewVTWidth = NewVT.getSizeInBits();
4127  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4128  Align, MMOFlags, AAInfo);
4129  LdChain.push_back(LdOp.getValue(1));
4130 
4131  // Check if we can load the element with one instruction.
4132  if (LdWidth <= NewVTWidth) {
4133  if (!NewVT.isVector()) {
4134  unsigned NumElts = WidenWidth / NewVTWidth;
4135  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4136  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4137  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4138  }
4139  if (NewVT == WidenVT)
4140  return LdOp;
4141 
4142  assert(WidenWidth % NewVTWidth == 0);
4143  unsigned NumConcat = WidenWidth / NewVTWidth;
4144  SmallVector<SDValue, 16> ConcatOps(NumConcat);
4145  SDValue UndefVal = DAG.getUNDEF(NewVT);
4146  ConcatOps[0] = LdOp;
4147  for (unsigned i = 1; i != NumConcat; ++i)
4148  ConcatOps[i] = UndefVal;
4149  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4150  }
4151 
4152  // Load vector by using multiple loads from largest vector to scalar.
4154  LdOps.push_back(LdOp);
4155 
4156  LdWidth -= NewVTWidth;
4157  unsigned Offset = 0;
4158 
4159  while (LdWidth > 0) {
4160  unsigned Increment = NewVTWidth / 8;
4161  Offset += Increment;
4162  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4163 
4164  SDValue L;
4165  if (LdWidth < NewVTWidth) {
4166  // The current type we are using is too large. Find a better size.
4167  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4168  NewVTWidth = NewVT.getSizeInBits();
4169  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4170  LD->getPointerInfo().getWithOffset(Offset),
4171  MinAlign(Align, Increment), MMOFlags, AAInfo);
4172  LdChain.push_back(L.getValue(1));
4173  if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
4174  // Later code assumes the vector loads produced will be mergeable, so we
4175  // must pad the final entry up to the previous width. Scalars are
4176  // combined separately.
4178  Loads.push_back(L);
4179  unsigned size = L->getValueSizeInBits(0);
4180  while (size < LdOp->getValueSizeInBits(0)) {
4181  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4182  size += L->getValueSizeInBits(0);
4183  }
4184  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4185  }
4186  } else {
4187  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4188  LD->getPointerInfo().getWithOffset(Offset),
4189  MinAlign(Align, Increment), MMOFlags, AAInfo);
4190  LdChain.push_back(L.getValue(1));
4191  }
4192 
4193  LdOps.push_back(L);
4194  LdOp = L;
4195 
4196  LdWidth -= NewVTWidth;
4197  }
4198 
4199  // Build the vector from the load operations.
4200  unsigned End = LdOps.size();
4201  if (!LdOps[0].getValueType().isVector())
4202  // All the loads are scalar loads.
4203  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4204 
4205  // If the load contains vectors, build the vector using concat vector.
4206  // All of the vectors used to load are power-of-2, and the scalar loads can be
4207  // combined to make a power-of-2 vector.
4208  SmallVector<SDValue, 16> ConcatOps(End);
4209  int i = End - 1;
4210  int Idx = End;
4211  EVT LdTy = LdOps[i].getValueType();
4212  // First, combine the scalar loads to a vector.
4213  if (!LdTy.isVector()) {
4214  for (--i; i >= 0; --i) {
4215  LdTy = LdOps[i].getValueType();
4216  if (LdTy.isVector())
4217  break;
4218  }
4219  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4220  }
4221  ConcatOps[--Idx] = LdOps[i];
4222  for (--i; i >= 0; --i) {
4223  EVT NewLdTy = LdOps[i].getValueType();
4224  if (NewLdTy != LdTy) {
4225  // Create a larger vector.
4226  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4227  makeArrayRef(&ConcatOps[Idx], End - Idx));
4228  Idx = End - 1;
4229  LdTy = NewLdTy;
4230  }
4231  ConcatOps[--Idx] = LdOps[i];
4232  }
4233 
4234  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
4235  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4236  makeArrayRef(&ConcatOps[Idx], End - Idx));
4237 
4238  // We need to fill the rest with undefs to build the vector.
4239  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
4240  SmallVector<SDValue, 16> WidenOps(NumOps);
4241  SDValue UndefVal = DAG.getUNDEF(LdTy);
4242  {
4243  unsigned i = 0;
4244  for (; i != End-Idx; ++i)
4245  WidenOps[i] = ConcatOps[Idx+i];
4246  for (; i != NumOps; ++i)
4247  WidenOps[i] = UndefVal;
4248  }
4249  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
4250 }
4251 
4252 SDValue
4253 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
4254  LoadSDNode *LD,
4256  // For extension loads, it may not be more efficient to chop up the vector
4257  // and then extend it. Instead, we unroll the load and build a new vector.
4258  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4259  EVT LdVT = LD->getMemoryVT();
4260  SDLoc dl(LD);
4261  assert(LdVT.isVector() && WidenVT.isVector());
4262 
4263  // Load information
4264  SDValue Chain = LD->getChain();
4265  SDValue BasePtr = LD->getBasePtr();
4266  unsigned Align = LD->getAlignment();
4267  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4268  AAMDNodes AAInfo = LD->getAAInfo();
4269 
4270  EVT EltVT = WidenVT.getVectorElementType();
4271  EVT LdEltVT = LdVT.getVectorElementType();
4272  unsigned NumElts = LdVT.getVectorNumElements();
4273 
4274  // Load each element and widen.
4275  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4276  SmallVector<SDValue, 16> Ops(WidenNumElts);
4277  unsigned Increment = LdEltVT.getSizeInBits() / 8;
4278  Ops[0] =
4279  DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
4280  LdEltVT, Align, MMOFlags, AAInfo);
4281  LdChain.push_back(Ops[0].getValue(1));
4282  unsigned i = 0, Offset = Increment;
4283  for (i=1; i < NumElts; ++i, Offset += Increment) {
4284  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4285  Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
4286  LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
4287  Align, MMOFlags, AAInfo);
4288  LdChain.push_back(Ops[i].getValue(1));
4289  }
4290 
4291  // Fill the rest with undefs.
4292  SDValue UndefVal = DAG.getUNDEF(EltVT);
4293  for (; i != WidenNumElts; ++i)
4294  Ops[i] = UndefVal;
4295 
4296  return DAG.getBuildVector(WidenVT, dl, Ops);
4297 }
4298 
4299 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
4300  StoreSDNode *ST) {
4301  // The strategy assumes that we can efficiently store power-of-two widths.
4302  // The routine chops the vector into the largest vector stores with the same
4303  // element type or scalar stores.
4304  SDValue Chain = ST->getChain();
4305  SDValue BasePtr = ST->getBasePtr();
4306  unsigned Align = ST->getAlignment();
4307  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4308  AAMDNodes AAInfo = ST->getAAInfo();
4309  SDValue ValOp = GetWidenedVector(ST->getValue());
4310  SDLoc dl(ST);
4311 
4312  EVT StVT = ST->getMemoryVT();
4313  unsigned StWidth = StVT.getSizeInBits();
4314  EVT ValVT = ValOp.getValueType();
4315  unsigned ValWidth = ValVT.getSizeInBits();
4316  EVT ValEltVT = ValVT.getVectorElementType();
4317  unsigned ValEltWidth = ValEltVT.getSizeInBits();
4318  assert(StVT.getVectorElementType() == ValEltVT);
4319 
4320  int Idx = 0; // current index to store
4321  unsigned Offset = 0; // offset from base to store
4322  while (StWidth != 0) {
4323  // Find the largest vector type we can store with.
4324  EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
4325  unsigned NewVTWidth = NewVT.getSizeInBits();
4326  unsigned Increment = NewVTWidth / 8;
4327  if (NewVT.isVector()) {
4328  unsigned NumVTElts = NewVT.getVectorNumElements();
4329  do {
4330  SDValue EOp = DAG.getNode(
4331  ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
4332  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4333  StChain.push_back(DAG.getStore(
4334  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4335  MinAlign(Align, Offset), MMOFlags, AAInfo));
4336  StWidth -= NewVTWidth;
4337  Offset += Increment;
4338  Idx += NumVTElts;
4339 
4340  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4341  } while (StWidth != 0 && StWidth >= NewVTWidth);
4342  } else {
4343  // Cast the vector to the scalar type we can store.
4344  unsigned NumElts = ValWidth / NewVTWidth;
4345  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4346  SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4347  // Readjust index position based on new vector type.
4348  Idx = Idx * ValEltWidth / NewVTWidth;
4349  do {
4350  SDValue EOp = DAG.getNode(
4351  ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4352  DAG.getConstant(Idx++, dl,
4353  TLI.getVectorIdxTy(DAG.getDataLayout())));
4354  StChain.push_back(DAG.getStore(
4355  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4356  MinAlign(Align, Offset), MMOFlags, AAInfo));
4357  StWidth -= NewVTWidth;
4358  Offset += Increment;
4359  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4360  } while (StWidth != 0 && StWidth >= NewVTWidth);
4361  // Restore index back to be relative to the original widen element type.
4362  Idx = Idx * NewVTWidth / ValEltWidth;
4363  }
4364  }
4365 }
4366 
4367 void
4368 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4369  StoreSDNode *ST) {
4370  // For extension loads, it may not be more efficient to truncate the vector
4371  // and then store it. Instead, we extract each element and then store it.
4372  SDValue Chain = ST->getChain();
4373  SDValue BasePtr = ST->getBasePtr();
4374  unsigned Align = ST->getAlignment();
4375  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4376  AAMDNodes AAInfo = ST->getAAInfo();
4377  SDValue ValOp = GetWidenedVector(ST->getValue());
4378  SDLoc dl(ST);
4379 
4380  EVT StVT = ST->getMemoryVT();
4381  EVT ValVT = ValOp.getValueType();
4382 
4383  // It must be true that the wide vector type is bigger than where we need to
4384  // store.
4385  assert(StVT.isVector() && ValOp.getValueType().isVector());
4386  assert(StVT.bitsLT(ValOp.getValueType()));
4387 
4388  // For truncating stores, we can not play the tricks of chopping legal vector
4389  // types and bitcast it to the right type. Instead, we unroll the store.
4390  EVT StEltVT = StVT.getVectorElementType();
4391  EVT ValEltVT = ValVT.getVectorElementType();
4392  unsigned Increment = ValEltVT.getSizeInBits() / 8;
4393  unsigned NumElts = StVT.getVectorNumElements();
4394  SDValue EOp = DAG.getNode(
4395  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4396  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4397  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4398  ST->getPointerInfo(), StEltVT, Align,
4399  MMOFlags, AAInfo));
4400  unsigned Offset = Increment;
4401  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4402  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4403  SDValue EOp = DAG.getNode(
4404  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4405  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4406  StChain.push_back(DAG.getTruncStore(
4407  Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4408  StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4409  }
4410 }
4411 
4412 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
4413 /// input vector must have the same element type as NVT.
4414 /// FillWithZeroes specifies that the vector should be widened with zeroes.
4415 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
4416  bool FillWithZeroes) {
4417  // Note that InOp might have been widened so it might already have
4418  // the right width or it might need be narrowed.
4419  EVT InVT = InOp.getValueType();
4421  "input and widen element type must match");
4422  SDLoc dl(InOp);
4423 
4424  // Check if InOp already has the right width.
4425  if (InVT == NVT)
4426  return InOp;
4427 
4428  unsigned InNumElts = InVT.getVectorNumElements();
4429  unsigned WidenNumElts = NVT.getVectorNumElements();
4430  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
4431  unsigned NumConcat = WidenNumElts / InNumElts;
4432  SmallVector<SDValue, 16> Ops(NumConcat);
4433  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
4434  DAG.getUNDEF(InVT);
4435  Ops[0] = InOp;
4436  for (unsigned i = 1; i != NumConcat; ++i)
4437  Ops[i] = FillVal;
4438 
4439  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
4440  }
4441 
4442  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
4443  return DAG.getNode(
4444  ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
4445  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4446 
4447  // Fall back to extract and build.
4448  SmallVector<SDValue, 16> Ops(WidenNumElts);
4449  EVT EltVT = NVT.getVectorElementType();
4450  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
4451  unsigned Idx;
4452  for (Idx = 0; Idx < MinNumElts; ++Idx)
4453  Ops[Idx] = DAG.getNode(
4454  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4455  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4456 
4457  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
4458  DAG.getUNDEF(EltVT);
4459  for ( ; Idx < WidenNumElts; ++Idx)
4460  Ops[Idx] = FillVal;
4461  return DAG.getBuildVector(NVT, dl, Ops);
4462 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:549
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:516
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:572
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isUndef() const
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:281
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:342
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, SmallVectorImpl< SDValue > &ConcatOps, unsigned ConcatEnd, EVT VT, EVT MaxVT, EVT WidenVT)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:351
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
#define LLVM_FALLTHROUGH
Definition: Compiler.h:86
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:497
SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:351
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getChain() const
unsigned getAlignment() const
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:274
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:966
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:488
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:368
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:337
uint64_t High
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:415
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
A description of a memory reference used in the backend.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:390
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
This is used by foldAnyOrAllBitsSet() to capture a source value (Root) and the bit indexes (Mask) nee...
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
const SDValue & getPassThru() const
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:457
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:398
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:458
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
const SDValue & getScale() const
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:409
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:867
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:573
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:475
const SDValue & getMask() const
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:502
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:311
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:229
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:610
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getBasePtr() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
These reductions are non-strict, and have a single vector operand.
Definition: ISDOpcodes.h:846
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:429
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:69
Simple binary floating point operators.
Definition: ISDOpcodes.h:268
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:912
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:316
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node&#39;s operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:732
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:434
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition: ValueTypes.h:58
bool hasNoNaNs() const
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
const SDValue & getValue() const
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:396
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:534
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
size_t size() const
Definition: SmallVector.h:53
bool isVolatile() const
This class contains a discriminated un