LLVM  9.0.0svn
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
23 #include "llvm/IR/DataLayout.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Result Vector Scalarization: <1 x ty> -> ty.
32 //===----------------------------------------------------------------------===//
33 
34 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
35  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
36  dbgs() << "\n");
37  SDValue R = SDValue();
38 
39  switch (N->getOpcode()) {
40  default:
41 #ifndef NDEBUG
42  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
43  N->dump(&DAG);
44  dbgs() << "\n";
45 #endif
46  report_fatal_error("Do not know how to scalarize the result of this "
47  "operator!\n");
48 
49  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
50  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
51  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
52  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
53  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
54  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
55  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
56  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
57  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
58  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
59  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
60  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
61  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
62  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
63  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
64  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
65  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
69  R = ScalarizeVecRes_VecInregOp(N);
70  break;
71  case ISD::ANY_EXTEND:
72  case ISD::BITREVERSE:
73  case ISD::BSWAP:
74  case ISD::CTLZ:
76  case ISD::CTPOP:
77  case ISD::CTTZ:
79  case ISD::FABS:
80  case ISD::FCEIL:
81  case ISD::FCOS:
82  case ISD::FEXP:
83  case ISD::FEXP2:
84  case ISD::FFLOOR:
85  case ISD::FLOG:
86  case ISD::FLOG10:
87  case ISD::FLOG2:
88  case ISD::FNEARBYINT:
89  case ISD::FNEG:
90  case ISD::FP_EXTEND:
91  case ISD::FP_TO_SINT:
92  case ISD::FP_TO_UINT:
93  case ISD::FRINT:
94  case ISD::FROUND:
95  case ISD::FSIN:
96  case ISD::FSQRT:
97  case ISD::FTRUNC:
98  case ISD::SIGN_EXTEND:
99  case ISD::SINT_TO_FP:
100  case ISD::TRUNCATE:
101  case ISD::UINT_TO_FP:
102  case ISD::ZERO_EXTEND:
103  case ISD::FCANONICALIZE:
104  R = ScalarizeVecRes_UnaryOp(N);
105  break;
106 
107  case ISD::ADD:
108  case ISD::AND:
109  case ISD::FADD:
110  case ISD::FCOPYSIGN:
111  case ISD::FDIV:
112  case ISD::FMUL:
113  case ISD::FMINNUM:
114  case ISD::FMAXNUM:
115  case ISD::FMINNUM_IEEE:
116  case ISD::FMAXNUM_IEEE:
117  case ISD::FMINIMUM:
118  case ISD::FMAXIMUM:
119  case ISD::SMIN:
120  case ISD::SMAX:
121  case ISD::UMIN:
122  case ISD::UMAX:
123 
124  case ISD::SADDSAT:
125  case ISD::UADDSAT:
126  case ISD::SSUBSAT:
127  case ISD::USUBSAT:
128 
129  case ISD::FPOW:
130  case ISD::FREM:
131  case ISD::FSUB:
132  case ISD::MUL:
133  case ISD::OR:
134  case ISD::SDIV:
135  case ISD::SREM:
136  case ISD::SUB:
137  case ISD::UDIV:
138  case ISD::UREM:
139  case ISD::XOR:
140  case ISD::SHL:
141  case ISD::SRA:
142  case ISD::SRL:
143  R = ScalarizeVecRes_BinOp(N);
144  break;
145  case ISD::FMA:
146  R = ScalarizeVecRes_TernaryOp(N);
147  break;
148  case ISD::STRICT_FADD:
149  case ISD::STRICT_FSUB:
150  case ISD::STRICT_FMUL:
151  case ISD::STRICT_FDIV:
152  case ISD::STRICT_FREM:
153  case ISD::STRICT_FSQRT:
154  case ISD::STRICT_FMA:
155  case ISD::STRICT_FPOW:
156  case ISD::STRICT_FPOWI:
157  case ISD::STRICT_FSIN:
158  case ISD::STRICT_FCOS:
159  case ISD::STRICT_FEXP:
160  case ISD::STRICT_FEXP2:
161  case ISD::STRICT_FLOG:
162  case ISD::STRICT_FLOG10:
163  case ISD::STRICT_FLOG2:
164  case ISD::STRICT_FRINT:
166  case ISD::STRICT_FMAXNUM:
167  case ISD::STRICT_FMINNUM:
168  case ISD::STRICT_FCEIL:
169  case ISD::STRICT_FFLOOR:
170  case ISD::STRICT_FROUND:
171  case ISD::STRICT_FTRUNC:
172  R = ScalarizeVecRes_StrictFPOp(N);
173  break;
174  case ISD::UADDO:
175  case ISD::SADDO:
176  case ISD::USUBO:
177  case ISD::SSUBO:
178  case ISD::UMULO:
179  case ISD::SMULO:
180  R = ScalarizeVecRes_OverflowOp(N, ResNo);
181  break;
182  case ISD::SMULFIX:
183  case ISD::UMULFIX:
184  R = ScalarizeVecRes_MULFIX(N);
185  break;
186  }
187 
188  // If R is null, the sub-method took care of registering the result.
189  if (R.getNode())
190  SetScalarizedVector(SDValue(N, ResNo), R);
191 }
192 
193 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
194  SDValue LHS = GetScalarizedVector(N->getOperand(0));
195  SDValue RHS = GetScalarizedVector(N->getOperand(1));
196  return DAG.getNode(N->getOpcode(), SDLoc(N),
197  LHS.getValueType(), LHS, RHS, N->getFlags());
198 }
199 
200 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
201  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
202  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
203  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
204  return DAG.getNode(N->getOpcode(), SDLoc(N),
205  Op0.getValueType(), Op0, Op1, Op2);
206 }
207 
208 SDValue DAGTypeLegalizer::ScalarizeVecRes_MULFIX(SDNode *N) {
209  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
210  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
211  SDValue Op2 = N->getOperand(2);
212  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
213  Op2);
214 }
215 
216 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
218  unsigned NumOpers = N->getNumOperands();
219  SDValue Chain = N->getOperand(0);
220  EVT ValueVTs[] = {VT, MVT::Other};
221  SDLoc dl(N);
222 
224 
225  // The Chain is the first operand.
226  Opers.push_back(Chain);
227 
228  // Now process the remaining operands.
229  for (unsigned i = 1; i < NumOpers; ++i) {
230  SDValue Oper = N->getOperand(i);
231 
232  if (Oper.getValueType().isVector())
233  Oper = GetScalarizedVector(Oper);
234 
235  Opers.push_back(Oper);
236  }
237 
238  SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
239 
240  // Legalize the chain result - switch anything that used the old chain to
241  // use the new one.
242  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
243  return Result;
244 }
245 
246 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
247  unsigned ResNo) {
248  SDLoc DL(N);
249  EVT ResVT = N->getValueType(0);
250  EVT OvVT = N->getValueType(1);
251 
252  SDValue ScalarLHS, ScalarRHS;
253  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
254  ScalarLHS = GetScalarizedVector(N->getOperand(0));
255  ScalarRHS = GetScalarizedVector(N->getOperand(1));
256  } else {
257  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
258  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
259  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
260  ScalarLHS = ElemsLHS[0];
261  ScalarRHS = ElemsRHS[0];
262  }
263 
264  SDVTList ScalarVTs = DAG.getVTList(
266  SDNode *ScalarNode = DAG.getNode(
267  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
268 
269  // Replace the other vector result not being explicitly scalarized here.
270  unsigned OtherNo = 1 - ResNo;
271  EVT OtherVT = N->getValueType(OtherNo);
272  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
273  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
274  } else {
275  SDValue OtherVal = DAG.getNode(
276  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
277  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
278  }
279 
280  return SDValue(ScalarNode, ResNo);
281 }
282 
283 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
284  unsigned ResNo) {
285  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
286  return GetScalarizedVector(Op);
287 }
288 
289 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
290  SDValue Op = N->getOperand(0);
291  if (Op.getValueType().isVector()
292  && Op.getValueType().getVectorNumElements() == 1
293  && !isSimpleLegalType(Op.getValueType()))
294  Op = GetScalarizedVector(Op);
295  EVT NewVT = N->getValueType(0).getVectorElementType();
296  return DAG.getNode(ISD::BITCAST, SDLoc(N),
297  NewVT, Op);
298 }
299 
300 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
301  EVT EltVT = N->getValueType(0).getVectorElementType();
302  SDValue InOp = N->getOperand(0);
303  // The BUILD_VECTOR operands may be of wider element types and
304  // we may need to truncate them back to the requested return type.
305  if (EltVT.isInteger())
306  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
307  return InOp;
308 }
309 
310 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
311  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
313  N->getOperand(0), N->getOperand(1));
314 }
315 
316 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
317  EVT NewVT = N->getValueType(0).getVectorElementType();
318  SDValue Op = GetScalarizedVector(N->getOperand(0));
319  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
320  NewVT, Op, N->getOperand(1));
321 }
322 
323 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
324  SDValue Op = GetScalarizedVector(N->getOperand(0));
325  return DAG.getNode(ISD::FPOWI, SDLoc(N),
326  Op.getValueType(), Op, N->getOperand(1));
327 }
328 
329 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
330  // The value to insert may have a wider type than the vector element type,
331  // so be sure to truncate it to the element type if necessary.
332  SDValue Op = N->getOperand(1);
333  EVT EltVT = N->getValueType(0).getVectorElementType();
334  if (Op.getValueType() != EltVT)
335  // FIXME: Can this happen for floating point types?
336  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
337  return Op;
338 }
339 
340 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
341  assert(N->isUnindexed() && "Indexed vector load?");
342 
343  SDValue Result = DAG.getLoad(
346  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
349  N->getAAInfo());
350 
351  // Legalize the chain result - switch anything that used the old chain to
352  // use the new one.
353  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
354  return Result;
355 }
356 
357 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
358  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
359  EVT DestVT = N->getValueType(0).getVectorElementType();
360  SDValue Op = N->getOperand(0);
361  EVT OpVT = Op.getValueType();
362  SDLoc DL(N);
363  // The result needs scalarizing, but it's not a given that the source does.
364  // This is a workaround for targets where it's impossible to scalarize the
365  // result of a conversion, because the source type is legal.
366  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
367  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
368  // legal and was not scalarized.
369  // See the similar logic in ScalarizeVecRes_SETCC
370  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
371  Op = GetScalarizedVector(Op);
372  } else {
373  EVT VT = OpVT.getVectorElementType();
374  Op = DAG.getNode(
375  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
376  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
377  }
378  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
379 }
380 
381 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
382  EVT EltVT = N->getValueType(0).getVectorElementType();
383  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
384  SDValue LHS = GetScalarizedVector(N->getOperand(0));
385  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
386  LHS, DAG.getValueType(ExtVT));
387 }
388 
389 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
390  SDLoc DL(N);
391  SDValue Op = N->getOperand(0);
392 
393  EVT OpVT = Op.getValueType();
394  EVT OpEltVT = OpVT.getVectorElementType();
395  EVT EltVT = N->getValueType(0).getVectorElementType();
396 
397  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
398  Op = GetScalarizedVector(Op);
399  } else {
400  Op = DAG.getNode(
401  ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
402  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
403  }
404 
405  switch (N->getOpcode()) {
407  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
409  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
411  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
412  }
413 
414  llvm_unreachable("Illegal extend_vector_inreg opcode");
415 }
416 
417 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
418  // If the operand is wider than the vector element type then it is implicitly
419  // truncated. Make that explicit here.
420  EVT EltVT = N->getValueType(0).getVectorElementType();
421  SDValue InOp = N->getOperand(0);
422  if (InOp.getValueType() != EltVT)
423  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
424  return InOp;
425 }
426 
427 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
428  SDValue Cond = N->getOperand(0);
429  EVT OpVT = Cond.getValueType();
430  SDLoc DL(N);
431  // The vselect result and true/value operands needs scalarizing, but it's
432  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
433  // See the similar logic in ScalarizeVecRes_SETCC
434  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
435  Cond = GetScalarizedVector(Cond);
436  } else {
437  EVT VT = OpVT.getVectorElementType();
438  Cond = DAG.getNode(
439  ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
440  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
441  }
442 
443  SDValue LHS = GetScalarizedVector(N->getOperand(1));
444  TargetLowering::BooleanContent ScalarBool =
445  TLI.getBooleanContents(false, false);
446  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
447 
448  // If integer and float booleans have different contents then we can't
449  // reliably optimize in all cases. There is a full explanation for this in
450  // DAGCombiner::visitSELECT() where the same issue affects folding
451  // (select C, 0, 1) to (xor C, 1).
452  if (TLI.getBooleanContents(false, false) !=
453  TLI.getBooleanContents(false, true)) {
454  // At least try the common case where the boolean is generated by a
455  // comparison.
456  if (Cond->getOpcode() == ISD::SETCC) {
457  EVT OpVT = Cond->getOperand(0).getValueType();
458  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
459  VecBool = TLI.getBooleanContents(OpVT);
460  } else
462  }
463 
464  EVT CondVT = Cond.getValueType();
465  if (ScalarBool != VecBool) {
466  switch (ScalarBool) {
468  break;
472  // Vector read from all ones, scalar expects a single 1 so mask.
473  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
474  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
475  break;
479  // Vector reads from a one, scalar from all ones so sign extend.
480  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
481  Cond, DAG.getValueType(MVT::i1));
482  break;
483  }
484  }
485 
486  // Truncate the condition if needed
487  auto BoolVT = getSetCCResultType(CondVT);
488  if (BoolVT.bitsLT(CondVT))
489  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
490 
491  return DAG.getSelect(SDLoc(N),
492  LHS.getValueType(), Cond, LHS,
493  GetScalarizedVector(N->getOperand(2)));
494 }
495 
496 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
497  SDValue LHS = GetScalarizedVector(N->getOperand(1));
498  return DAG.getSelect(SDLoc(N),
499  LHS.getValueType(), N->getOperand(0), LHS,
500  GetScalarizedVector(N->getOperand(2)));
501 }
502 
503 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
504  SDValue LHS = GetScalarizedVector(N->getOperand(2));
505  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
506  N->getOperand(0), N->getOperand(1),
507  LHS, GetScalarizedVector(N->getOperand(3)),
508  N->getOperand(4));
509 }
510 
511 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
512  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
513 }
514 
515 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
516  // Figure out if the scalar is the LHS or RHS and return it.
517  SDValue Arg = N->getOperand(2).getOperand(0);
518  if (Arg.isUndef())
519  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
520  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
521  return GetScalarizedVector(N->getOperand(Op));
522 }
523 
524 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
525  assert(N->getValueType(0).isVector() &&
526  N->getOperand(0).getValueType().isVector() &&
527  "Operand types must be vectors");
528  SDValue LHS = N->getOperand(0);
529  SDValue RHS = N->getOperand(1);
530  EVT OpVT = LHS.getValueType();
531  EVT NVT = N->getValueType(0).getVectorElementType();
532  SDLoc DL(N);
533 
534  // The result needs scalarizing, but it's not a given that the source does.
535  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
536  LHS = GetScalarizedVector(LHS);
537  RHS = GetScalarizedVector(RHS);
538  } else {
539  EVT VT = OpVT.getVectorElementType();
540  LHS = DAG.getNode(
541  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
542  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
543  RHS = DAG.getNode(
544  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
545  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
546  }
547 
548  // Turn it into a scalar SETCC.
549  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
550  N->getOperand(2));
551  // Vectors may have a different boolean contents to scalars. Promote the
552  // value appropriately.
553  ISD::NodeType ExtendCode =
555  return DAG.getNode(ExtendCode, DL, NVT, Res);
556 }
557 
558 
559 //===----------------------------------------------------------------------===//
560 // Operand Vector Scalarization <1 x ty> -> ty.
561 //===----------------------------------------------------------------------===//
562 
563 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
564  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
565  dbgs() << "\n");
566  SDValue Res = SDValue();
567 
568  if (!Res.getNode()) {
569  switch (N->getOpcode()) {
570  default:
571 #ifndef NDEBUG
572  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
573  N->dump(&DAG);
574  dbgs() << "\n";
575 #endif
576  report_fatal_error("Do not know how to scalarize this operator's "
577  "operand!\n");
578  case ISD::BITCAST:
579  Res = ScalarizeVecOp_BITCAST(N);
580  break;
581  case ISD::ANY_EXTEND:
582  case ISD::ZERO_EXTEND:
583  case ISD::SIGN_EXTEND:
584  case ISD::TRUNCATE:
585  case ISD::FP_TO_SINT:
586  case ISD::FP_TO_UINT:
587  case ISD::SINT_TO_FP:
588  case ISD::UINT_TO_FP:
589  Res = ScalarizeVecOp_UnaryOp(N);
590  break;
591  case ISD::CONCAT_VECTORS:
592  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
593  break;
595  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
596  break;
597  case ISD::VSELECT:
598  Res = ScalarizeVecOp_VSELECT(N);
599  break;
600  case ISD::SETCC:
601  Res = ScalarizeVecOp_VSETCC(N);
602  break;
603  case ISD::STORE:
604  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
605  break;
606  case ISD::FP_ROUND:
607  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
608  break;
609  }
610  }
611 
612  // If the result is null, the sub-method took care of registering results etc.
613  if (!Res.getNode()) return false;
614 
615  // If the result is N, the sub-method updated N in place. Tell the legalizer
616  // core about this.
617  if (Res.getNode() == N)
618  return true;
619 
620  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
621  "Invalid operand expansion");
622 
623  ReplaceValueWith(SDValue(N, 0), Res);
624  return false;
625 }
626 
627 /// If the value to convert is a vector that needs to be scalarized, it must be
628 /// <1 x ty>. Convert the element instead.
629 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
630  SDValue Elt = GetScalarizedVector(N->getOperand(0));
631  return DAG.getNode(ISD::BITCAST, SDLoc(N),
632  N->getValueType(0), Elt);
633 }
634 
635 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
636 /// Do the operation on the element instead.
637 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
639  "Unexpected vector type!");
640  SDValue Elt = GetScalarizedVector(N->getOperand(0));
641  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
642  N->getValueType(0).getScalarType(), Elt);
643  // Revectorize the result so the types line up with what the uses of this
644  // expression expect.
645  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
646 }
647 
648 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
649 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
651  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
652  Ops[i] = GetScalarizedVector(N->getOperand(i));
653  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
654 }
655 
656 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
657 /// so just return the element, ignoring the index.
658 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
659  EVT VT = N->getValueType(0);
660  SDValue Res = GetScalarizedVector(N->getOperand(0));
661  if (Res.getValueType() != VT)
662  Res = VT.isFloatingPoint()
663  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
664  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
665  return Res;
666 }
667 
668 /// If the input condition is a vector that needs to be scalarized, it must be
669 /// <1 x i1>, so just convert to a normal ISD::SELECT
670 /// (still with vector output type since that was acceptable if we got here).
671 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
672  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
673  EVT VT = N->getValueType(0);
674 
675  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
676  N->getOperand(2));
677 }
678 
679 /// If the operand is a vector that needs to be scalarized then the
680 /// result must be v1i1, so just convert to a scalar SETCC and wrap
681 /// with a scalar_to_vector since the res type is legal if we got here
682 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
683  assert(N->getValueType(0).isVector() &&
684  N->getOperand(0).getValueType().isVector() &&
685  "Operand types must be vectors");
686  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
687 
688  EVT VT = N->getValueType(0);
689  SDValue LHS = GetScalarizedVector(N->getOperand(0));
690  SDValue RHS = GetScalarizedVector(N->getOperand(1));
691 
692  EVT OpVT = N->getOperand(0).getValueType();
693  EVT NVT = VT.getVectorElementType();
694  SDLoc DL(N);
695  // Turn it into a scalar SETCC.
696  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
697  N->getOperand(2));
698 
699  // Vectors may have a different boolean contents to scalars. Promote the
700  // value appropriately.
701  ISD::NodeType ExtendCode =
703 
704  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
705 
706  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
707 }
708 
709 /// If the value to store is a vector that needs to be scalarized, it must be
710 /// <1 x ty>. Just store the element.
711 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
712  assert(N->isUnindexed() && "Indexed store of one-element vector?");
713  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
714  SDLoc dl(N);
715 
716  if (N->isTruncatingStore())
717  return DAG.getTruncStore(
718  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
719  N->getBasePtr(), N->getPointerInfo(),
721  N->getMemOperand()->getFlags(), N->getAAInfo());
722 
723  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
724  N->getBasePtr(), N->getPointerInfo(),
726  N->getAAInfo());
727 }
728 
729 /// If the value to round is a vector that needs to be scalarized, it must be
730 /// <1 x ty>. Convert the element instead.
731 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
732  SDValue Elt = GetScalarizedVector(N->getOperand(0));
733  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
734  N->getValueType(0).getVectorElementType(), Elt,
735  N->getOperand(1));
736  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
737 }
738 
739 //===----------------------------------------------------------------------===//
740 // Result Vector Splitting
741 //===----------------------------------------------------------------------===//
742 
743 /// This method is called when the specified result of the specified node is
744 /// found to need vector splitting. At this point, the node may also have
745 /// invalid operands or may have other results that need legalization, we just
746 /// know that (at least) one result needs vector splitting.
747 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
748  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
749  SDValue Lo, Hi;
750 
751  // See if the target wants to custom expand this node.
752  if (CustomLowerNode(N, N->getValueType(ResNo), true))
753  return;
754 
755  switch (N->getOpcode()) {
756  default:
757 #ifndef NDEBUG
758  dbgs() << "SplitVectorResult #" << ResNo << ": ";
759  N->dump(&DAG);
760  dbgs() << "\n";
761 #endif
762  report_fatal_error("Do not know how to split the result of this "
763  "operator!\n");
764 
765  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
766  case ISD::VSELECT:
767  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
768  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
769  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
770  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
771  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
772  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
773  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
774  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
775  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
776  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
777  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
778  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
779  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
780  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
781  case ISD::LOAD:
782  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
783  break;
784  case ISD::MLOAD:
785  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
786  break;
787  case ISD::MGATHER:
788  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
789  break;
790  case ISD::SETCC:
791  SplitVecRes_SETCC(N, Lo, Hi);
792  break;
793  case ISD::VECTOR_SHUFFLE:
794  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
795  break;
796 
800  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
801  break;
802 
803  case ISD::BITREVERSE:
804  case ISD::BSWAP:
805  case ISD::CTLZ:
806  case ISD::CTTZ:
809  case ISD::CTPOP:
810  case ISD::FABS:
811  case ISD::FCEIL:
812  case ISD::FCOS:
813  case ISD::FEXP:
814  case ISD::FEXP2:
815  case ISD::FFLOOR:
816  case ISD::FLOG:
817  case ISD::FLOG10:
818  case ISD::FLOG2:
819  case ISD::FNEARBYINT:
820  case ISD::FNEG:
821  case ISD::FP_EXTEND:
822  case ISD::FP_ROUND:
823  case ISD::FP_TO_SINT:
824  case ISD::FP_TO_UINT:
825  case ISD::FRINT:
826  case ISD::FROUND:
827  case ISD::FSIN:
828  case ISD::FSQRT:
829  case ISD::FTRUNC:
830  case ISD::SINT_TO_FP:
831  case ISD::TRUNCATE:
832  case ISD::UINT_TO_FP:
833  case ISD::FCANONICALIZE:
834  SplitVecRes_UnaryOp(N, Lo, Hi);
835  break;
836 
837  case ISD::ANY_EXTEND:
838  case ISD::SIGN_EXTEND:
839  case ISD::ZERO_EXTEND:
840  SplitVecRes_ExtendOp(N, Lo, Hi);
841  break;
842 
843  case ISD::ADD:
844  case ISD::SUB:
845  case ISD::MUL:
846  case ISD::MULHS:
847  case ISD::MULHU:
848  case ISD::FADD:
849  case ISD::FSUB:
850  case ISD::FMUL:
851  case ISD::FMINNUM:
852  case ISD::FMAXNUM:
853  case ISD::FMINIMUM:
854  case ISD::FMAXIMUM:
855  case ISD::SDIV:
856  case ISD::UDIV:
857  case ISD::FDIV:
858  case ISD::FPOW:
859  case ISD::AND:
860  case ISD::OR:
861  case ISD::XOR:
862  case ISD::SHL:
863  case ISD::SRA:
864  case ISD::SRL:
865  case ISD::UREM:
866  case ISD::SREM:
867  case ISD::FREM:
868  case ISD::SMIN:
869  case ISD::SMAX:
870  case ISD::UMIN:
871  case ISD::UMAX:
872  case ISD::SADDSAT:
873  case ISD::UADDSAT:
874  case ISD::SSUBSAT:
875  case ISD::USUBSAT:
876  SplitVecRes_BinOp(N, Lo, Hi);
877  break;
878  case ISD::FMA:
879  SplitVecRes_TernaryOp(N, Lo, Hi);
880  break;
881  case ISD::STRICT_FADD:
882  case ISD::STRICT_FSUB:
883  case ISD::STRICT_FMUL:
884  case ISD::STRICT_FDIV:
885  case ISD::STRICT_FREM:
886  case ISD::STRICT_FSQRT:
887  case ISD::STRICT_FMA:
888  case ISD::STRICT_FPOW:
889  case ISD::STRICT_FPOWI:
890  case ISD::STRICT_FSIN:
891  case ISD::STRICT_FCOS:
892  case ISD::STRICT_FEXP:
893  case ISD::STRICT_FEXP2:
894  case ISD::STRICT_FLOG:
895  case ISD::STRICT_FLOG10:
896  case ISD::STRICT_FLOG2:
897  case ISD::STRICT_FRINT:
899  case ISD::STRICT_FMAXNUM:
900  case ISD::STRICT_FMINNUM:
901  case ISD::STRICT_FCEIL:
902  case ISD::STRICT_FFLOOR:
903  case ISD::STRICT_FROUND:
904  case ISD::STRICT_FTRUNC:
905  SplitVecRes_StrictFPOp(N, Lo, Hi);
906  break;
907  case ISD::UADDO:
908  case ISD::SADDO:
909  case ISD::USUBO:
910  case ISD::SSUBO:
911  case ISD::UMULO:
912  case ISD::SMULO:
913  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
914  break;
915  case ISD::SMULFIX:
916  case ISD::UMULFIX:
917  SplitVecRes_MULFIX(N, Lo, Hi);
918  break;
919  }
920 
921  // If Lo/Hi is null, the sub-method took care of registering results etc.
922  if (Lo.getNode())
923  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
924 }
925 
926 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
927  SDValue &Hi) {
928  SDValue LHSLo, LHSHi;
929  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
930  SDValue RHSLo, RHSHi;
931  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
932  SDLoc dl(N);
933 
934  const SDNodeFlags Flags = N->getFlags();
935  unsigned Opcode = N->getOpcode();
936  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
937  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
938 }
939 
940 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
941  SDValue &Hi) {
942  SDValue Op0Lo, Op0Hi;
943  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
944  SDValue Op1Lo, Op1Hi;
945  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
946  SDValue Op2Lo, Op2Hi;
947  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
948  SDLoc dl(N);
949 
950  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
951  Op0Lo, Op1Lo, Op2Lo);
952  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
953  Op0Hi, Op1Hi, Op2Hi);
954 }
955 
956 void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
957  SDValue LHSLo, LHSHi;
958  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
959  SDValue RHSLo, RHSHi;
960  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
961  SDLoc dl(N);
962  SDValue Op2 = N->getOperand(2);
963 
964  unsigned Opcode = N->getOpcode();
965  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
966  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
967 }
968 
969 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
970  SDValue &Hi) {
971  // We know the result is a vector. The input may be either a vector or a
972  // scalar value.
973  EVT LoVT, HiVT;
974  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
975  SDLoc dl(N);
976 
977  SDValue InOp = N->getOperand(0);
978  EVT InVT = InOp.getValueType();
979 
980  // Handle some special cases efficiently.
981  switch (getTypeAction(InVT)) {
988  break;
991  // A scalar to vector conversion, where the scalar needs expansion.
992  // If the vector is being split in two then we can just convert the
993  // expanded pieces.
994  if (LoVT == HiVT) {
995  GetExpandedOp(InOp, Lo, Hi);
996  if (DAG.getDataLayout().isBigEndian())
997  std::swap(Lo, Hi);
998  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
999  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1000  return;
1001  }
1002  break;
1004  // If the input is a vector that needs to be split, convert each split
1005  // piece of the input now.
1006  GetSplitVector(InOp, Lo, Hi);
1007  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1008  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1009  return;
1010  }
1011 
1012  // In the general case, convert the input to an integer and split it by hand.
1013  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1014  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1015  if (DAG.getDataLayout().isBigEndian())
1016  std::swap(LoIntVT, HiIntVT);
1017 
1018  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1019 
1020  if (DAG.getDataLayout().isBigEndian())
1021  std::swap(Lo, Hi);
1022  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1023  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1024 }
1025 
1026 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1027  SDValue &Hi) {
1028  EVT LoVT, HiVT;
1029  SDLoc dl(N);
1030  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1031  unsigned LoNumElts = LoVT.getVectorNumElements();
1032  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1033  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1034 
1035  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1036  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1037 }
1038 
1039 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1040  SDValue &Hi) {
1041  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1042  SDLoc dl(N);
1043  unsigned NumSubvectors = N->getNumOperands() / 2;
1044  if (NumSubvectors == 1) {
1045  Lo = N->getOperand(0);
1046  Hi = N->getOperand(1);
1047  return;
1048  }
1049 
1050  EVT LoVT, HiVT;
1051  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1052 
1053  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1054  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1055 
1056  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1057  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1058 }
1059 
1060 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1061  SDValue &Hi) {
1062  SDValue Vec = N->getOperand(0);
1063  SDValue Idx = N->getOperand(1);
1064  SDLoc dl(N);
1065 
1066  EVT LoVT, HiVT;
1067  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1068 
1069  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1070  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1071  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1072  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
1073  TLI.getVectorIdxTy(DAG.getDataLayout())));
1074 }
1075 
1076 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1077  SDValue &Hi) {
1078  SDValue Vec = N->getOperand(0);
1079  SDValue SubVec = N->getOperand(1);
1080  SDValue Idx = N->getOperand(2);
1081  SDLoc dl(N);
1082  GetSplitVector(Vec, Lo, Hi);
1083 
1084  EVT VecVT = Vec.getValueType();
1085  unsigned VecElems = VecVT.getVectorNumElements();
1086  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
1087 
1088  // If we know the index is 0, and we know the subvector doesn't cross the
1089  // boundary between the halves, we can avoid spilling the vector, and insert
1090  // into the lower half of the split vector directly.
1091  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
1092  // the index is constant and there is no boundary crossing. But those cases
1093  // don't seem to get hit in practice.
1094  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
1095  unsigned IdxVal = ConstIdx->getZExtValue();
1096  if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
1097  EVT LoVT, HiVT;
1098  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1099  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1100  return;
1101  }
1102  }
1103 
1104  // Spill the vector to the stack.
1105  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1106  SDValue Store =
1107  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1108 
1109  // Store the new subvector into the specified index.
1110  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1111  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1112  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1113  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1114 
1115  // Load the Lo part from the stack slot.
1116  Lo =
1117  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1118 
1119  // Increment the pointer to the other part.
1120  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1121  StackPtr =
1122  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1123  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
1124 
1125  // Load the Hi part from the stack slot.
1126  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1127  MinAlign(Alignment, IncrementSize));
1128 }
1129 
1130 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1131  SDValue &Hi) {
1132  SDLoc dl(N);
1133  GetSplitVector(N->getOperand(0), Lo, Hi);
1134  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1135  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1136 }
1137 
1138 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1139  SDValue &Hi) {
1140  SDValue LHSLo, LHSHi;
1141  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1142  SDLoc DL(N);
1143 
1144  SDValue RHSLo, RHSHi;
1145  SDValue RHS = N->getOperand(1);
1146  EVT RHSVT = RHS.getValueType();
1147  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1148  GetSplitVector(RHS, RHSLo, RHSHi);
1149  else
1150  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1151 
1152 
1153  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1154  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1155 }
1156 
1157 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1158  SDValue &Hi) {
1159  SDValue LHSLo, LHSHi;
1160  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1161  SDLoc dl(N);
1162 
1163  EVT LoVT, HiVT;
1164  std::tie(LoVT, HiVT) =
1165  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1166 
1167  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1168  DAG.getValueType(LoVT));
1169  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1170  DAG.getValueType(HiVT));
1171 }
1172 
1173 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1174  SDValue &Hi) {
1175  unsigned Opcode = N->getOpcode();
1176  SDValue N0 = N->getOperand(0);
1177 
1178  SDLoc dl(N);
1179  SDValue InLo, InHi;
1180 
1181  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1182  GetSplitVector(N0, InLo, InHi);
1183  else
1184  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1185 
1186  EVT InLoVT = InLo.getValueType();
1187  unsigned InNumElements = InLoVT.getVectorNumElements();
1188 
1189  EVT OutLoVT, OutHiVT;
1190  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1191  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1192  assert((2 * OutNumElements) <= InNumElements &&
1193  "Illegal extend vector in reg split");
1194 
1195  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1196  // input vector (i.e. we only use InLo):
1197  // OutLo will extend the first OutNumElements from InLo.
1198  // OutHi will extend the next OutNumElements from InLo.
1199 
1200  // Shuffle the elements from InLo for OutHi into the bottom elements to
1201  // create a 'fake' InHi.
1202  SmallVector<int, 8> SplitHi(InNumElements, -1);
1203  for (unsigned i = 0; i != OutNumElements; ++i)
1204  SplitHi[i] = i + OutNumElements;
1205  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1206 
1207  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1208  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1209 }
1210 
1211 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1212  SDValue &Hi) {
1213  unsigned NumOps = N->getNumOperands();
1214  SDValue Chain = N->getOperand(0);
1215  EVT LoVT, HiVT;
1216  SDLoc dl(N);
1217  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1218 
1221 
1222  // The Chain is the first operand.
1223  OpsLo.push_back(Chain);
1224  OpsHi.push_back(Chain);
1225 
1226  // Now process the remaining operands.
1227  for (unsigned i = 1; i < NumOps; ++i) {
1228  SDValue Op = N->getOperand(i);
1229  SDValue OpLo = Op;
1230  SDValue OpHi = Op;
1231 
1232  EVT InVT = Op.getValueType();
1233  if (InVT.isVector()) {
1234  // If the input also splits, handle it directly for a
1235  // compile time speedup. Otherwise split it by hand.
1236  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1237  GetSplitVector(Op, OpLo, OpHi);
1238  else
1239  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1240  }
1241 
1242  OpsLo.push_back(OpLo);
1243  OpsHi.push_back(OpHi);
1244  }
1245 
1246  EVT LoValueVTs[] = {LoVT, MVT::Other};
1247  EVT HiValueVTs[] = {HiVT, MVT::Other};
1248  Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1249  Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1250 
1251  // Build a factor node to remember that this Op is independent of the
1252  // other one.
1253  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1254  Lo.getValue(1), Hi.getValue(1));
1255 
1256  // Legalize the chain result - switch anything that used the old chain to
1257  // use the new one.
1258  ReplaceValueWith(SDValue(N, 1), Chain);
1259 }
1260 
1261 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1262  SDValue &Lo, SDValue &Hi) {
1263  SDLoc dl(N);
1264  EVT ResVT = N->getValueType(0);
1265  EVT OvVT = N->getValueType(1);
1266  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1267  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1268  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1269 
1270  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1271  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1272  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1273  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1274  } else {
1275  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1276  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1277  }
1278 
1279  unsigned Opcode = N->getOpcode();
1280  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1281  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1282  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1283  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1284 
1285  Lo = SDValue(LoNode, ResNo);
1286  Hi = SDValue(HiNode, ResNo);
1287 
1288  // Replace the other vector result not being explicitly split here.
1289  unsigned OtherNo = 1 - ResNo;
1290  EVT OtherVT = N->getValueType(OtherNo);
1291  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1292  SetSplitVector(SDValue(N, OtherNo),
1293  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1294  } else {
1295  SDValue OtherVal = DAG.getNode(
1296  ISD::CONCAT_VECTORS, dl, OtherVT,
1297  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1298  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1299  }
1300 }
1301 
1302 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1303  SDValue &Hi) {
1304  SDValue Vec = N->getOperand(0);
1305  SDValue Elt = N->getOperand(1);
1306  SDValue Idx = N->getOperand(2);
1307  SDLoc dl(N);
1308  GetSplitVector(Vec, Lo, Hi);
1309 
1310  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1311  unsigned IdxVal = CIdx->getZExtValue();
1312  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1313  if (IdxVal < LoNumElts)
1314  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1315  Lo.getValueType(), Lo, Elt, Idx);
1316  else
1317  Hi =
1318  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1319  DAG.getConstant(IdxVal - LoNumElts, dl,
1320  TLI.getVectorIdxTy(DAG.getDataLayout())));
1321  return;
1322  }
1323 
1324  // See if the target wants to custom expand this node.
1325  if (CustomLowerNode(N, N->getValueType(0), true))
1326  return;
1327 
1328  // Make the vector elements byte-addressable if they aren't already.
1329  EVT VecVT = Vec.getValueType();
1330  EVT EltVT = VecVT.getVectorElementType();
1331  if (VecVT.getScalarSizeInBits() < 8) {
1332  EltVT = MVT::i8;
1333  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1334  VecVT.getVectorNumElements());
1335  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1336  // Extend the element type to match if needed.
1337  if (EltVT.bitsGT(Elt.getValueType()))
1338  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1339  }
1340 
1341  // Spill the vector to the stack.
1342  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1343  auto &MF = DAG.getMachineFunction();
1344  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1345  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1346  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1347 
1348  // Store the new element. This may be larger than the vector element type,
1349  // so use a truncating store.
1350  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1351  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1352  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1353  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1355 
1356  EVT LoVT, HiVT;
1357  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1358 
1359  // Load the Lo part from the stack slot.
1360  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1361 
1362  // Increment the pointer to the other part.
1363  unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1364  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1365  DAG.getConstant(IncrementSize, dl,
1366  StackPtr.getValueType()));
1367 
1368  // Load the Hi part from the stack slot.
1369  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1370  PtrInfo.getWithOffset(IncrementSize),
1371  MinAlign(Alignment, IncrementSize));
1372 
1373  // If we adjusted the original type, we need to truncate the results.
1374  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1375  if (LoVT != Lo.getValueType())
1376  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1377  if (HiVT != Hi.getValueType())
1378  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1379 }
1380 
1381 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1382  SDValue &Hi) {
1383  EVT LoVT, HiVT;
1384  SDLoc dl(N);
1385  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1386  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1387  Hi = DAG.getUNDEF(HiVT);
1388 }
1389 
1390 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1391  SDValue &Hi) {
1392  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1393  EVT LoVT, HiVT;
1394  SDLoc dl(LD);
1395  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1396 
1398  SDValue Ch = LD->getChain();
1399  SDValue Ptr = LD->getBasePtr();
1400  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1401  EVT MemoryVT = LD->getMemoryVT();
1402  unsigned Alignment = LD->getOriginalAlignment();
1403  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1404  AAMDNodes AAInfo = LD->getAAInfo();
1405 
1406  EVT LoMemVT, HiMemVT;
1407  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1408 
1409  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1410  LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1411 
1412  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1413  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1414  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1415  LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1416  Alignment, MMOFlags, AAInfo);
1417 
1418  // Build a factor node to remember that this load is independent of the
1419  // other one.
1420  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1421  Hi.getValue(1));
1422 
1423  // Legalize the chain result - switch anything that used the old chain to
1424  // use the new one.
1425  ReplaceValueWith(SDValue(LD, 1), Ch);
1426 }
1427 
1428 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1429  SDValue &Lo, SDValue &Hi) {
1430  EVT LoVT, HiVT;
1431  SDLoc dl(MLD);
1432  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1433 
1434  SDValue Ch = MLD->getChain();
1435  SDValue Ptr = MLD->getBasePtr();
1436  SDValue Mask = MLD->getMask();
1437  SDValue PassThru = MLD->getPassThru();
1438  unsigned Alignment = MLD->getOriginalAlignment();
1440 
1441  // if Alignment is equal to the vector size,
1442  // take the half of it for the second part
1443  unsigned SecondHalfAlignment =
1444  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1445  Alignment/2 : Alignment;
1446 
1447  // Split Mask operand
1448  SDValue MaskLo, MaskHi;
1449  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1450  GetSplitVector(Mask, MaskLo, MaskHi);
1451  else
1452  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1453 
1454  EVT MemoryVT = MLD->getMemoryVT();
1455  EVT LoMemVT, HiMemVT;
1456  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1457 
1458  SDValue PassThruLo, PassThruHi;
1459  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1460  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1461  else
1462  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1463 
1465  getMachineMemOperand(MLD->getPointerInfo(),
1467  Alignment, MLD->getAAInfo(), MLD->getRanges());
1468 
1469  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
1470  ExtType, MLD->isExpandingLoad());
1471 
1472  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1473  MLD->isExpandingLoad());
1474  unsigned HiOffset = LoMemVT.getStoreSize();
1475 
1478  HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
1479  MLD->getRanges());
1480 
1481  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
1482  ExtType, MLD->isExpandingLoad());
1483 
1484  // Build a factor node to remember that this load is independent of the
1485  // other one.
1486  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1487  Hi.getValue(1));
1488 
1489  // Legalize the chain result - switch anything that used the old chain to
1490  // use the new one.
1491  ReplaceValueWith(SDValue(MLD, 1), Ch);
1492 
1493 }
1494 
1495 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1496  SDValue &Lo, SDValue &Hi) {
1497  EVT LoVT, HiVT;
1498  SDLoc dl(MGT);
1499  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1500 
1501  SDValue Ch = MGT->getChain();
1502  SDValue Ptr = MGT->getBasePtr();
1503  SDValue Mask = MGT->getMask();
1504  SDValue PassThru = MGT->getPassThru();
1505  SDValue Index = MGT->getIndex();
1506  SDValue Scale = MGT->getScale();
1507  unsigned Alignment = MGT->getOriginalAlignment();
1508 
1509  // Split Mask operand
1510  SDValue MaskLo, MaskHi;
1511  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1512  GetSplitVector(Mask, MaskLo, MaskHi);
1513  else
1514  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1515 
1516  EVT MemoryVT = MGT->getMemoryVT();
1517  EVT LoMemVT, HiMemVT;
1518  // Split MemoryVT
1519  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1520 
1521  SDValue PassThruLo, PassThruHi;
1522  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1523  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1524  else
1525  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1526 
1527  SDValue IndexHi, IndexLo;
1528  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1529  GetSplitVector(Index, IndexLo, IndexHi);
1530  else
1531  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1532 
1534  getMachineMemOperand(MGT->getPointerInfo(),
1536  Alignment, MGT->getAAInfo(), MGT->getRanges());
1537 
1538  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1539  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1540  MMO);
1541 
1542  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1543  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1544  MMO);
1545 
1546  // Build a factor node to remember that this load is independent of the
1547  // other one.
1548  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1549  Hi.getValue(1));
1550 
1551  // Legalize the chain result - switch anything that used the old chain to
1552  // use the new one.
1553  ReplaceValueWith(SDValue(MGT, 1), Ch);
1554 }
1555 
1556 
1557 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1558  assert(N->getValueType(0).isVector() &&
1559  N->getOperand(0).getValueType().isVector() &&
1560  "Operand types must be vectors");
1561 
1562  EVT LoVT, HiVT;
1563  SDLoc DL(N);
1564  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1565 
1566  // If the input also splits, handle it directly. Otherwise split it by hand.
1567  SDValue LL, LH, RL, RH;
1568  if (getTypeAction(N->getOperand(0).getValueType()) ==
1570  GetSplitVector(N->getOperand(0), LL, LH);
1571  else
1572  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1573 
1574  if (getTypeAction(N->getOperand(1).getValueType()) ==
1576  GetSplitVector(N->getOperand(1), RL, RH);
1577  else
1578  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1579 
1580  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1581  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1582 }
1583 
1584 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1585  SDValue &Hi) {
1586  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1587  EVT LoVT, HiVT;
1588  SDLoc dl(N);
1589  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1590 
1591  // If the input also splits, handle it directly for a compile time speedup.
1592  // Otherwise split it by hand.
1593  EVT InVT = N->getOperand(0).getValueType();
1594  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1595  GetSplitVector(N->getOperand(0), Lo, Hi);
1596  else
1597  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1598 
1599  if (N->getOpcode() == ISD::FP_ROUND) {
1600  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1601  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1602  } else {
1603  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1604  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1605  }
1606 }
1607 
1608 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1609  SDValue &Hi) {
1610  SDLoc dl(N);
1611  EVT SrcVT = N->getOperand(0).getValueType();
1612  EVT DestVT = N->getValueType(0);
1613  EVT LoVT, HiVT;
1614  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1615 
1616  // We can do better than a generic split operation if the extend is doing
1617  // more than just doubling the width of the elements and the following are
1618  // true:
1619  // - The number of vector elements is even,
1620  // - the source type is legal,
1621  // - the type of a split source is illegal,
1622  // - the type of an extended (by doubling element size) source is legal, and
1623  // - the type of that extended source when split is legal.
1624  //
1625  // This won't necessarily completely legalize the operation, but it will
1626  // more effectively move in the right direction and prevent falling down
1627  // to scalarization in many cases due to the input vector being split too
1628  // far.
1629  unsigned NumElements = SrcVT.getVectorNumElements();
1630  if ((NumElements & 1) == 0 &&
1631  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1632  LLVMContext &Ctx = *DAG.getContext();
1633  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1634  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1635 
1636  EVT SplitLoVT, SplitHiVT;
1637  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1638  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1639  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1640  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1641  N->dump(&DAG); dbgs() << "\n");
1642  // Extend the source vector by one step.
1643  SDValue NewSrc =
1644  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1645  // Get the low and high halves of the new, extended one step, vector.
1646  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1647  // Extend those vector halves the rest of the way.
1648  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1649  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1650  return;
1651  }
1652  }
1653  // Fall back to the generic unary operator splitting otherwise.
1654  SplitVecRes_UnaryOp(N, Lo, Hi);
1655 }
1656 
1657 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1658  SDValue &Lo, SDValue &Hi) {
1659  // The low and high parts of the original input give four input vectors.
1660  SDValue Inputs[4];
1661  SDLoc dl(N);
1662  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1663  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1664  EVT NewVT = Inputs[0].getValueType();
1665  unsigned NewElts = NewVT.getVectorNumElements();
1666 
1667  // If Lo or Hi uses elements from at most two of the four input vectors, then
1668  // express it as a vector shuffle of those two inputs. Otherwise extract the
1669  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1671  for (unsigned High = 0; High < 2; ++High) {
1672  SDValue &Output = High ? Hi : Lo;
1673 
1674  // Build a shuffle mask for the output, discovering on the fly which
1675  // input vectors to use as shuffle operands (recorded in InputUsed).
1676  // If building a suitable shuffle vector proves too hard, then bail
1677  // out with useBuildVector set.
1678  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1679  unsigned FirstMaskIdx = High * NewElts;
1680  bool useBuildVector = false;
1681  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1682  // The mask element. This indexes into the input.
1683  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1684 
1685  // The input vector this mask element indexes into.
1686  unsigned Input = (unsigned)Idx / NewElts;
1687 
1688  if (Input >= array_lengthof(Inputs)) {
1689  // The mask element does not index into any input vector.
1690  Ops.push_back(-1);
1691  continue;
1692  }
1693 
1694  // Turn the index into an offset from the start of the input vector.
1695  Idx -= Input * NewElts;
1696 
1697  // Find or create a shuffle vector operand to hold this input.
1698  unsigned OpNo;
1699  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1700  if (InputUsed[OpNo] == Input) {
1701  // This input vector is already an operand.
1702  break;
1703  } else if (InputUsed[OpNo] == -1U) {
1704  // Create a new operand for this input vector.
1705  InputUsed[OpNo] = Input;
1706  break;
1707  }
1708  }
1709 
1710  if (OpNo >= array_lengthof(InputUsed)) {
1711  // More than two input vectors used! Give up on trying to create a
1712  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1713  useBuildVector = true;
1714  break;
1715  }
1716 
1717  // Add the mask index for the new shuffle vector.
1718  Ops.push_back(Idx + OpNo * NewElts);
1719  }
1720 
1721  if (useBuildVector) {
1722  EVT EltVT = NewVT.getVectorElementType();
1724 
1725  // Extract the input elements by hand.
1726  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1727  // The mask element. This indexes into the input.
1728  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1729 
1730  // The input vector this mask element indexes into.
1731  unsigned Input = (unsigned)Idx / NewElts;
1732 
1733  if (Input >= array_lengthof(Inputs)) {
1734  // The mask element is "undef" or indexes off the end of the input.
1735  SVOps.push_back(DAG.getUNDEF(EltVT));
1736  continue;
1737  }
1738 
1739  // Turn the index into an offset from the start of the input vector.
1740  Idx -= Input * NewElts;
1741 
1742  // Extract the vector element by hand.
1743  SVOps.push_back(DAG.getNode(
1744  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1745  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1746  }
1747 
1748  // Construct the Lo/Hi output using a BUILD_VECTOR.
1749  Output = DAG.getBuildVector(NewVT, dl, SVOps);
1750  } else if (InputUsed[0] == -1U) {
1751  // No input vectors were used! The result is undefined.
1752  Output = DAG.getUNDEF(NewVT);
1753  } else {
1754  SDValue Op0 = Inputs[InputUsed[0]];
1755  // If only one input was used, use an undefined vector for the other.
1756  SDValue Op1 = InputUsed[1] == -1U ?
1757  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1758  // At least one input vector was used. Create a new shuffle vector.
1759  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1760  }
1761 
1762  Ops.clear();
1763  }
1764 }
1765 
1766 
1767 //===----------------------------------------------------------------------===//
1768 // Operand Vector Splitting
1769 //===----------------------------------------------------------------------===//
1770 
1771 /// This method is called when the specified operand of the specified node is
1772 /// found to need vector splitting. At this point, all of the result types of
1773 /// the node are known to be legal, but other operands of the node may need
1774 /// legalization as well as the specified one.
1775 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1776  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1777  SDValue Res = SDValue();
1778 
1779  // See if the target wants to custom split this node.
1780  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1781  return false;
1782 
1783  if (!Res.getNode()) {
1784  switch (N->getOpcode()) {
1785  default:
1786 #ifndef NDEBUG
1787  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1788  N->dump(&DAG);
1789  dbgs() << "\n";
1790 #endif
1791  report_fatal_error("Do not know how to split this operator's "
1792  "operand!\n");
1793 
1794  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1795  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1796  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1797  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1798  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1799  case ISD::TRUNCATE:
1800  Res = SplitVecOp_TruncateHelper(N);
1801  break;
1802  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1803  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1804  case ISD::STORE:
1805  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1806  break;
1807  case ISD::MSTORE:
1808  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1809  break;
1810  case ISD::MSCATTER:
1811  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1812  break;
1813  case ISD::MGATHER:
1814  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1815  break;
1816  case ISD::VSELECT:
1817  Res = SplitVecOp_VSELECT(N, OpNo);
1818  break;
1819  case ISD::SINT_TO_FP:
1820  case ISD::UINT_TO_FP:
1821  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1822  Res = SplitVecOp_TruncateHelper(N);
1823  else
1824  Res = SplitVecOp_UnaryOp(N);
1825  break;
1826  case ISD::FP_TO_SINT:
1827  case ISD::FP_TO_UINT:
1828  case ISD::CTTZ:
1829  case ISD::CTLZ:
1830  case ISD::CTPOP:
1831  case ISD::FP_EXTEND:
1832  case ISD::SIGN_EXTEND:
1833  case ISD::ZERO_EXTEND:
1834  case ISD::ANY_EXTEND:
1835  case ISD::FTRUNC:
1836  case ISD::FCANONICALIZE:
1837  Res = SplitVecOp_UnaryOp(N);
1838  break;
1839 
1843  Res = SplitVecOp_ExtVecInRegOp(N);
1844  break;
1845 
1846  case ISD::VECREDUCE_FADD:
1847  case ISD::VECREDUCE_FMUL:
1848  case ISD::VECREDUCE_ADD:
1849  case ISD::VECREDUCE_MUL:
1850  case ISD::VECREDUCE_AND:
1851  case ISD::VECREDUCE_OR:
1852  case ISD::VECREDUCE_XOR:
1853  case ISD::VECREDUCE_SMAX:
1854  case ISD::VECREDUCE_SMIN:
1855  case ISD::VECREDUCE_UMAX:
1856  case ISD::VECREDUCE_UMIN:
1857  case ISD::VECREDUCE_FMAX:
1858  case ISD::VECREDUCE_FMIN:
1859  Res = SplitVecOp_VECREDUCE(N, OpNo);
1860  break;
1861  }
1862  }
1863 
1864  // If the result is null, the sub-method took care of registering results etc.
1865  if (!Res.getNode()) return false;
1866 
1867  // If the result is N, the sub-method updated N in place. Tell the legalizer
1868  // core about this.
1869  if (Res.getNode() == N)
1870  return true;
1871 
1872  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1873  "Invalid operand expansion");
1874 
1875  ReplaceValueWith(SDValue(N, 0), Res);
1876  return false;
1877 }
1878 
1879 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1880  // The only possibility for an illegal operand is the mask, since result type
1881  // legalization would have handled this node already otherwise.
1882  assert(OpNo == 0 && "Illegal operand must be mask");
1883 
1884  SDValue Mask = N->getOperand(0);
1885  SDValue Src0 = N->getOperand(1);
1886  SDValue Src1 = N->getOperand(2);
1887  EVT Src0VT = Src0.getValueType();
1888  SDLoc DL(N);
1889  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1890 
1891  SDValue Lo, Hi;
1892  GetSplitVector(N->getOperand(0), Lo, Hi);
1893  assert(Lo.getValueType() == Hi.getValueType() &&
1894  "Lo and Hi have differing types");
1895 
1896  EVT LoOpVT, HiOpVT;
1897  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1898  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1899 
1900  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1901  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1902  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1903  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1904 
1905  SDValue LoSelect =
1906  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1907  SDValue HiSelect =
1908  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1909 
1910  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1911 }
1912 
1913 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
1914  EVT ResVT = N->getValueType(0);
1915  SDValue Lo, Hi;
1916  SDLoc dl(N);
1917 
1918  SDValue VecOp = N->getOperand(OpNo);
1919  EVT VecVT = VecOp.getValueType();
1920  assert(VecVT.isVector() && "Can only split reduce vector operand");
1921  GetSplitVector(VecOp, Lo, Hi);
1922  EVT LoOpVT, HiOpVT;
1923  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
1924 
1925  bool NoNaN = N->getFlags().hasNoNaNs();
1926  unsigned CombineOpc = 0;
1927  switch (N->getOpcode()) {
1928  case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
1929  case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
1930  case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
1931  case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
1932  case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
1933  case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
1934  case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
1935  case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
1936  case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
1937  case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
1938  case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
1939  case ISD::VECREDUCE_FMAX:
1940  CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXIMUM;
1941  break;
1942  case ISD::VECREDUCE_FMIN:
1943  CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM;
1944  break;
1945  default:
1946  llvm_unreachable("Unexpected reduce ISD node");
1947  }
1948 
1949  // Use the appropriate scalar instruction on the split subvectors before
1950  // reducing the now partially reduced smaller vector.
1951  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
1952  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
1953 }
1954 
1955 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1956  // The result has a legal vector type, but the input needs splitting.
1957  EVT ResVT = N->getValueType(0);
1958  SDValue Lo, Hi;
1959  SDLoc dl(N);
1960  GetSplitVector(N->getOperand(0), Lo, Hi);
1961  EVT InVT = Lo.getValueType();
1962 
1963  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1964  InVT.getVectorNumElements());
1965 
1966  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1967  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1968 
1969  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1970 }
1971 
1972 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1973  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1974  // end up being split all the way down to individual components. Convert the
1975  // split pieces into integers and reassemble.
1976  SDValue Lo, Hi;
1977  GetSplitVector(N->getOperand(0), Lo, Hi);
1978  Lo = BitConvertToInteger(Lo);
1979  Hi = BitConvertToInteger(Hi);
1980 
1981  if (DAG.getDataLayout().isBigEndian())
1982  std::swap(Lo, Hi);
1983 
1984  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1985  JoinIntegers(Lo, Hi));
1986 }
1987 
1988 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1989  // We know that the extracted result type is legal.
1990  EVT SubVT = N->getValueType(0);
1991  SDValue Idx = N->getOperand(1);
1992  SDLoc dl(N);
1993  SDValue Lo, Hi;
1994  GetSplitVector(N->getOperand(0), Lo, Hi);
1995 
1996  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1997  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1998 
1999  if (IdxVal < LoElts) {
2000  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
2001  "Extracted subvector crosses vector split!");
2002  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2003  } else {
2004  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2005  DAG.getConstant(IdxVal - LoElts, dl,
2006  Idx.getValueType()));
2007  }
2008 }
2009 
2010 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2011  SDValue Vec = N->getOperand(0);
2012  SDValue Idx = N->getOperand(1);
2013  EVT VecVT = Vec.getValueType();
2014 
2015  if (isa<ConstantSDNode>(Idx)) {
2016  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2017 
2018  SDValue Lo, Hi;
2019  GetSplitVector(Vec, Lo, Hi);
2020 
2021  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2022 
2023  if (IdxVal < LoElts)
2024  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2025  return SDValue(DAG.UpdateNodeOperands(N, Hi,
2026  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2027  Idx.getValueType())), 0);
2028  }
2029 
2030  // See if the target wants to custom expand this node.
2031  if (CustomLowerNode(N, N->getValueType(0), true))
2032  return SDValue();
2033 
2034  // Make the vector elements byte-addressable if they aren't already.
2035  SDLoc dl(N);
2036  EVT EltVT = VecVT.getVectorElementType();
2037  if (VecVT.getScalarSizeInBits() < 8) {
2038  EltVT = MVT::i8;
2039  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2040  VecVT.getVectorNumElements());
2041  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2042  }
2043 
2044  // Store the vector to the stack.
2045  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
2046  auto &MF = DAG.getMachineFunction();
2047  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2048  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2049  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2050 
2051  // Load back the required element.
2052  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2053 
2054  // FIXME: This is to handle i1 vectors with elements promoted to i8.
2055  // i1 vector handling needs general improvement.
2056  if (N->getValueType(0).bitsLT(EltVT)) {
2057  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2059  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2060  }
2061 
2062  return DAG.getExtLoad(
2063  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2065 }
2066 
2067 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2068  SDValue Lo, Hi;
2069 
2070  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2071  // splitting the result has the same effect as splitting the input operand.
2072  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2073 
2074  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2075 }
2076 
2077 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2078  unsigned OpNo) {
2079  EVT LoVT, HiVT;
2080  SDLoc dl(MGT);
2081  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2082 
2083  SDValue Ch = MGT->getChain();
2084  SDValue Ptr = MGT->getBasePtr();
2085  SDValue Index = MGT->getIndex();
2086  SDValue Scale = MGT->getScale();
2087  SDValue Mask = MGT->getMask();
2088  SDValue PassThru = MGT->getPassThru();
2089  unsigned Alignment = MGT->getOriginalAlignment();
2090 
2091  SDValue MaskLo, MaskHi;
2092  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2093  // Split Mask operand
2094  GetSplitVector(Mask, MaskLo, MaskHi);
2095  else
2096  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2097 
2098  EVT MemoryVT = MGT->getMemoryVT();
2099  EVT LoMemVT, HiMemVT;
2100  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2101 
2102  SDValue PassThruLo, PassThruHi;
2103  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2104  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2105  else
2106  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2107 
2108  SDValue IndexHi, IndexLo;
2109  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2110  GetSplitVector(Index, IndexLo, IndexHi);
2111  else
2112  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2113 
2115  getMachineMemOperand(MGT->getPointerInfo(),
2117  Alignment, MGT->getAAInfo(), MGT->getRanges());
2118 
2119  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2120  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2121  OpsLo, MMO);
2122 
2123  MMO = DAG.getMachineFunction().
2124  getMachineMemOperand(MGT->getPointerInfo(),
2126  Alignment, MGT->getAAInfo(),
2127  MGT->getRanges());
2128 
2129  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2130  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2131  OpsHi, MMO);
2132 
2133  // Build a factor node to remember that this load is independent of the
2134  // other one.
2135  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2136  Hi.getValue(1));
2137 
2138  // Legalize the chain result - switch anything that used the old chain to
2139  // use the new one.
2140  ReplaceValueWith(SDValue(MGT, 1), Ch);
2141 
2142  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2143  Hi);
2144  ReplaceValueWith(SDValue(MGT, 0), Res);
2145  return SDValue();
2146 }
2147 
2148 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2149  unsigned OpNo) {
2150  SDValue Ch = N->getChain();
2151  SDValue Ptr = N->getBasePtr();
2152  SDValue Mask = N->getMask();
2153  SDValue Data = N->getValue();
2154  EVT MemoryVT = N->getMemoryVT();
2155  unsigned Alignment = N->getOriginalAlignment();
2156  SDLoc DL(N);
2157 
2158  EVT LoMemVT, HiMemVT;
2159  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2160 
2161  SDValue DataLo, DataHi;
2162  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2163  // Split Data operand
2164  GetSplitVector(Data, DataLo, DataHi);
2165  else
2166  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2167 
2168  SDValue MaskLo, MaskHi;
2169  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2170  // Split Mask operand
2171  GetSplitVector(Mask, MaskLo, MaskHi);
2172  else
2173  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2174 
2175  // if Alignment is equal to the vector size,
2176  // take the half of it for the second part
2177  unsigned SecondHalfAlignment =
2178  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
2179  Alignment/2 : Alignment;
2180 
2181  SDValue Lo, Hi;
2183  getMachineMemOperand(N->getPointerInfo(),
2185  Alignment, N->getAAInfo(), N->getRanges());
2186 
2187  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
2188  N->isTruncatingStore(),
2189  N->isCompressingStore());
2190 
2191  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2192  N->isCompressingStore());
2193  unsigned HiOffset = LoMemVT.getStoreSize();
2194 
2197  HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
2198  N->getRanges());
2199 
2200  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
2202 
2203  // Build a factor node to remember that this store is independent of the
2204  // other one.
2205  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2206 }
2207 
2208 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2209  unsigned OpNo) {
2210  SDValue Ch = N->getChain();
2211  SDValue Ptr = N->getBasePtr();
2212  SDValue Mask = N->getMask();
2213  SDValue Index = N->getIndex();
2214  SDValue Scale = N->getScale();
2215  SDValue Data = N->getValue();
2216  EVT MemoryVT = N->getMemoryVT();
2217  unsigned Alignment = N->getOriginalAlignment();
2218  SDLoc DL(N);
2219 
2220  // Split all operands
2221  EVT LoMemVT, HiMemVT;
2222  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2223 
2224  SDValue DataLo, DataHi;
2225  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2226  // Split Data operand
2227  GetSplitVector(Data, DataLo, DataHi);
2228  else
2229  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2230 
2231  SDValue MaskLo, MaskHi;
2232  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2233  // Split Mask operand
2234  GetSplitVector(Mask, MaskLo, MaskHi);
2235  else
2236  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2237 
2238  SDValue IndexHi, IndexLo;
2239  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2240  GetSplitVector(Index, IndexLo, IndexHi);
2241  else
2242  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2243 
2244  SDValue Lo;
2246  getMachineMemOperand(N->getPointerInfo(),
2248  Alignment, N->getAAInfo(), N->getRanges());
2249 
2250  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2251  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2252  DL, OpsLo, MMO);
2253 
2254  MMO = DAG.getMachineFunction().
2255  getMachineMemOperand(N->getPointerInfo(),
2257  Alignment, N->getAAInfo(), N->getRanges());
2258 
2259  // The order of the Scatter operation after split is well defined. The "Hi"
2260  // part comes after the "Lo". So these two operations should be chained one
2261  // after another.
2262  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2263  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2264  DL, OpsHi, MMO);
2265 }
2266 
2267 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2268  assert(N->isUnindexed() && "Indexed store of vector?");
2269  assert(OpNo == 1 && "Can only split the stored value");
2270  SDLoc DL(N);
2271 
2272  bool isTruncating = N->isTruncatingStore();
2273  SDValue Ch = N->getChain();
2274  SDValue Ptr = N->getBasePtr();
2275  EVT MemoryVT = N->getMemoryVT();
2276  unsigned Alignment = N->getOriginalAlignment();
2277  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2278  AAMDNodes AAInfo = N->getAAInfo();
2279  SDValue Lo, Hi;
2280  GetSplitVector(N->getOperand(1), Lo, Hi);
2281 
2282  EVT LoMemVT, HiMemVT;
2283  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2284 
2285  // Scalarize if the split halves are not byte-sized.
2286  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2287  return TLI.scalarizeVectorStore(N, DAG);
2288 
2289  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2290 
2291  if (isTruncating)
2292  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2293  Alignment, MMOFlags, AAInfo);
2294  else
2295  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2296  AAInfo);
2297 
2298  // Increment the pointer to the other half.
2299  Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2300 
2301  if (isTruncating)
2302  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2303  N->getPointerInfo().getWithOffset(IncrementSize),
2304  HiMemVT, Alignment, MMOFlags, AAInfo);
2305  else
2306  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2307  N->getPointerInfo().getWithOffset(IncrementSize),
2308  Alignment, MMOFlags, AAInfo);
2309 
2310  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2311 }
2312 
2313 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2314  SDLoc DL(N);
2315 
2316  // The input operands all must have the same type, and we know the result
2317  // type is valid. Convert this to a buildvector which extracts all the
2318  // input elements.
2319  // TODO: If the input elements are power-two vectors, we could convert this to
2320  // a new CONCAT_VECTORS node with elements that are half-wide.
2322  EVT EltVT = N->getValueType(0).getVectorElementType();
2323  for (const SDValue &Op : N->op_values()) {
2324  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2325  i != e; ++i) {
2326  Elts.push_back(DAG.getNode(
2327  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2328  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2329  }
2330  }
2331 
2332  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2333 }
2334 
2335 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2336  // The result type is legal, but the input type is illegal. If splitting
2337  // ends up with the result type of each half still being legal, just
2338  // do that. If, however, that would result in an illegal result type,
2339  // we can try to get more clever with power-two vectors. Specifically,
2340  // split the input type, but also widen the result element size, then
2341  // concatenate the halves and truncate again. For example, consider a target
2342  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2343  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2344  // %inlo = v4i32 extract_subvector %in, 0
2345  // %inhi = v4i32 extract_subvector %in, 4
2346  // %lo16 = v4i16 trunc v4i32 %inlo
2347  // %hi16 = v4i16 trunc v4i32 %inhi
2348  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2349  // %res = v8i8 trunc v8i16 %in16
2350  //
2351  // Without this transform, the original truncate would end up being
2352  // scalarized, which is pretty much always a last resort.
2353  SDValue InVec = N->getOperand(0);
2354  EVT InVT = InVec->getValueType(0);
2355  EVT OutVT = N->getValueType(0);
2356  unsigned NumElements = OutVT.getVectorNumElements();
2357  bool IsFloat = OutVT.isFloatingPoint();
2358 
2359  // Widening should have already made sure this is a power-two vector
2360  // if we're trying to split it at all. assert() that's true, just in case.
2361  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2362 
2363  unsigned InElementSize = InVT.getScalarSizeInBits();
2364  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2365 
2366  // Determine the split output VT. If its legal we can just split dirctly.
2367  EVT LoOutVT, HiOutVT;
2368  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2369  assert(LoOutVT == HiOutVT && "Unequal split?");
2370 
2371  // If the input elements are only 1/2 the width of the result elements,
2372  // just use the normal splitting. Our trick only work if there's room
2373  // to split more than once.
2374  if (isTypeLegal(LoOutVT) ||
2375  InElementSize <= OutElementSize * 2)
2376  return SplitVecOp_UnaryOp(N);
2377  SDLoc DL(N);
2378 
2379  // Don't touch if this will be scalarized.
2380  EVT FinalVT = InVT;
2381  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2382  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2383 
2384  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2385  return SplitVecOp_UnaryOp(N);
2386 
2387  // Get the split input vector.
2388  SDValue InLoVec, InHiVec;
2389  GetSplitVector(InVec, InLoVec, InHiVec);
2390 
2391  // Truncate them to 1/2 the element size.
2392  EVT HalfElementVT = IsFloat ?
2393  EVT::getFloatingPointVT(InElementSize/2) :
2394  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2395  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2396  NumElements/2);
2397  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2398  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2399  // Concatenate them to get the full intermediate truncation result.
2400  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2401  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2402  HalfHi);
2403  // Now finish up by truncating all the way down to the original result
2404  // type. This should normally be something that ends up being legal directly,
2405  // but in theory if a target has very wide vectors and an annoyingly
2406  // restricted set of legal types, this split can chain to build things up.
2407  return IsFloat
2408  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2409  DAG.getTargetConstant(
2410  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2411  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2412 }
2413 
2414 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2415  assert(N->getValueType(0).isVector() &&
2416  N->getOperand(0).getValueType().isVector() &&
2417  "Operand types must be vectors");
2418  // The result has a legal vector type, but the input needs splitting.
2419  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2420  SDLoc DL(N);
2421  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2422  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2423  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2424  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2425  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2426 
2427  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2428  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2429  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2430  return PromoteTargetBoolean(Con, N->getValueType(0));
2431 }
2432 
2433 
2434 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2435  // The result has a legal vector type, but the input needs splitting.
2436  EVT ResVT = N->getValueType(0);
2437  SDValue Lo, Hi;
2438  SDLoc DL(N);
2439  GetSplitVector(N->getOperand(0), Lo, Hi);
2440  EVT InVT = Lo.getValueType();
2441 
2442  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2443  InVT.getVectorNumElements());
2444 
2445  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2446  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2447 
2448  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2449 }
2450 
2451 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2452  // The result (and the first input) has a legal vector type, but the second
2453  // input needs splitting.
2454  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2455 }
2456 
2457 
2458 //===----------------------------------------------------------------------===//
2459 // Result Vector Widening
2460 //===----------------------------------------------------------------------===//
2461 
2462 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2463  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2464  dbgs() << "\n");
2465 
2466  // See if the target wants to custom widen this node.
2467  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2468  return;
2469 
2470  SDValue Res = SDValue();
2471  switch (N->getOpcode()) {
2472  default:
2473 #ifndef NDEBUG
2474  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2475  N->dump(&DAG);
2476  dbgs() << "\n";
2477 #endif
2478  llvm_unreachable("Do not know how to widen the result of this operator!");
2479 
2480  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2481  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2482  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2483  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2484  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2485  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2486  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2487  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2488  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2489  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2490  case ISD::VSELECT:
2491  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2492  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2493  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2494  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2495  case ISD::VECTOR_SHUFFLE:
2496  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2497  break;
2498  case ISD::MLOAD:
2499  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2500  break;
2501  case ISD::MGATHER:
2502  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2503  break;
2504 
2505  case ISD::ADD:
2506  case ISD::AND:
2507  case ISD::MUL:
2508  case ISD::MULHS:
2509  case ISD::MULHU:
2510  case ISD::OR:
2511  case ISD::SUB:
2512  case ISD::XOR:
2513  case ISD::FMINNUM:
2514  case ISD::FMAXNUM:
2515  case ISD::FMINIMUM:
2516  case ISD::FMAXIMUM:
2517  case ISD::SMIN:
2518  case ISD::SMAX:
2519  case ISD::UMIN:
2520  case ISD::UMAX:
2521  case ISD::UADDSAT:
2522  case ISD::SADDSAT:
2523  case ISD::USUBSAT:
2524  case ISD::SSUBSAT:
2525  Res = WidenVecRes_Binary(N);
2526  break;
2527 
2528  case ISD::FADD:
2529  case ISD::FMUL:
2530  case ISD::FPOW:
2531  case ISD::FSUB:
2532  case ISD::FDIV:
2533  case ISD::FREM:
2534  case ISD::SDIV:
2535  case ISD::UDIV:
2536  case ISD::SREM:
2537  case ISD::UREM:
2538  Res = WidenVecRes_BinaryCanTrap(N);
2539  break;
2540 
2541  case ISD::STRICT_FADD:
2542  case ISD::STRICT_FSUB:
2543  case ISD::STRICT_FMUL:
2544  case ISD::STRICT_FDIV:
2545  case ISD::STRICT_FREM:
2546  case ISD::STRICT_FSQRT:
2547  case ISD::STRICT_FMA:
2548  case ISD::STRICT_FPOW:
2549  case ISD::STRICT_FPOWI:
2550  case ISD::STRICT_FSIN:
2551  case ISD::STRICT_FCOS:
2552  case ISD::STRICT_FEXP:
2553  case ISD::STRICT_FEXP2:
2554  case ISD::STRICT_FLOG:
2555  case ISD::STRICT_FLOG10:
2556  case ISD::STRICT_FLOG2:
2557  case ISD::STRICT_FRINT:
2559  case ISD::STRICT_FMAXNUM:
2560  case ISD::STRICT_FMINNUM:
2561  case ISD::STRICT_FCEIL:
2562  case ISD::STRICT_FFLOOR:
2563  case ISD::STRICT_FROUND:
2564  case ISD::STRICT_FTRUNC:
2565  Res = WidenVecRes_StrictFP(N);
2566  break;
2567 
2568  case ISD::UADDO:
2569  case ISD::SADDO:
2570  case ISD::USUBO:
2571  case ISD::SSUBO:
2572  case ISD::UMULO:
2573  case ISD::SMULO:
2574  Res = WidenVecRes_OverflowOp(N, ResNo);
2575  break;
2576 
2577  case ISD::FCOPYSIGN:
2578  Res = WidenVecRes_FCOPYSIGN(N);
2579  break;
2580 
2581  case ISD::FPOWI:
2582  Res = WidenVecRes_POWI(N);
2583  break;
2584 
2585  case ISD::SHL:
2586  case ISD::SRA:
2587  case ISD::SRL:
2588  Res = WidenVecRes_Shift(N);
2589  break;
2590 
2594  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2595  break;
2596 
2597  case ISD::ANY_EXTEND:
2598  case ISD::FP_EXTEND:
2599  case ISD::FP_ROUND:
2600  case ISD::FP_TO_SINT:
2601  case ISD::FP_TO_UINT:
2602  case ISD::SIGN_EXTEND:
2603  case ISD::SINT_TO_FP:
2604  case ISD::TRUNCATE:
2605  case ISD::UINT_TO_FP:
2606  case ISD::ZERO_EXTEND:
2607  Res = WidenVecRes_Convert(N);
2608  break;
2609 
2610  case ISD::FABS:
2611  case ISD::FCEIL:
2612  case ISD::FCOS:
2613  case ISD::FEXP:
2614  case ISD::FEXP2:
2615  case ISD::FFLOOR:
2616  case ISD::FLOG:
2617  case ISD::FLOG10:
2618  case ISD::FLOG2:
2619  case ISD::FNEARBYINT:
2620  case ISD::FRINT:
2621  case ISD::FROUND:
2622  case ISD::FSIN:
2623  case ISD::FSQRT:
2624  case ISD::FTRUNC: {
2625  // We're going to widen this vector op to a legal type by padding with undef
2626  // elements. If the wide vector op is eventually going to be expanded to
2627  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2628  // libcalls on the undef elements. We are assuming that if the scalar op
2629  // requires expanding, then the vector op needs expanding too.
2630  EVT VT = N->getValueType(0);
2631  if (TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
2632  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2633  assert(!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2634  "Target supports vector op, but scalar requires expansion?");
2635  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2636  break;
2637  }
2638  }
2639  // If the target has custom/legal support for the scalar FP intrinsic ops
2640  // (they are probably not destined to become libcalls), then widen those like
2641  // any other unary ops.
2643 
2644  case ISD::BITREVERSE:
2645  case ISD::BSWAP:
2646  case ISD::CTLZ:
2647  case ISD::CTPOP:
2648  case ISD::CTTZ:
2649  case ISD::FNEG:
2650  case ISD::FCANONICALIZE:
2651  Res = WidenVecRes_Unary(N);
2652  break;
2653  case ISD::FMA:
2654  Res = WidenVecRes_Ternary(N);
2655  break;
2656  }
2657 
2658  // If Res is null, the sub-method took care of registering the result.
2659  if (Res.getNode())
2660  SetWidenedVector(SDValue(N, ResNo), Res);
2661 }
2662 
2663 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2664  // Ternary op widening.
2665  SDLoc dl(N);
2666  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2667  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2668  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2669  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2670  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2671 }
2672 
2673 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2674  // Binary op widening.
2675  SDLoc dl(N);
2676  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2677  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2678  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2679  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2680 }
2681 
2682 // Given a vector of operations that have been broken up to widen, see
2683 // if we can collect them together into the next widest legal VT. This
2684 // implementation is trap-safe.
2686  SmallVectorImpl<SDValue> &ConcatOps,
2687  unsigned ConcatEnd, EVT VT, EVT MaxVT,
2688  EVT WidenVT) {
2689  // Check to see if we have a single operation with the widen type.
2690  if (ConcatEnd == 1) {
2691  VT = ConcatOps[0].getValueType();
2692  if (VT == WidenVT)
2693  return ConcatOps[0];
2694  }
2695 
2696  SDLoc dl(ConcatOps[0]);
2697  EVT WidenEltVT = WidenVT.getVectorElementType();
2698  int Idx = 0;
2699 
2700  // while (Some element of ConcatOps is not of type MaxVT) {
2701  // From the end of ConcatOps, collect elements of the same type and put
2702  // them into an op of the next larger supported type
2703  // }
2704  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2705  Idx = ConcatEnd - 1;
2706  VT = ConcatOps[Idx--].getValueType();
2707  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2708  Idx--;
2709 
2710  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2711  EVT NextVT;
2712  do {
2713  NextSize *= 2;
2714  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2715  } while (!TLI.isTypeLegal(NextVT));
2716 
2717  if (!VT.isVector()) {
2718  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2719  SDValue VecOp = DAG.getUNDEF(NextVT);
2720  unsigned NumToInsert = ConcatEnd - Idx - 1;
2721  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2722  VecOp = DAG.getNode(
2723  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2724  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2725  }
2726  ConcatOps[Idx+1] = VecOp;
2727  ConcatEnd = Idx + 2;
2728  } else {
2729  // Vector type, create a CONCAT_VECTORS of type NextVT
2730  SDValue undefVec = DAG.getUNDEF(VT);
2731  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2732  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2733  unsigned RealVals = ConcatEnd - Idx - 1;
2734  unsigned SubConcatEnd = 0;
2735  unsigned SubConcatIdx = Idx + 1;
2736  while (SubConcatEnd < RealVals)
2737  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2738  while (SubConcatEnd < OpsToConcat)
2739  SubConcatOps[SubConcatEnd++] = undefVec;
2740  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2741  NextVT, SubConcatOps);
2742  ConcatEnd = SubConcatIdx + 1;
2743  }
2744  }
2745 
2746  // Check to see if we have a single operation with the widen type.
2747  if (ConcatEnd == 1) {
2748  VT = ConcatOps[0].getValueType();
2749  if (VT == WidenVT)
2750  return ConcatOps[0];
2751  }
2752 
2753  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2754  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2755  if (NumOps != ConcatEnd ) {
2756  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2757  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2758  ConcatOps[j] = UndefVal;
2759  }
2760  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2761  makeArrayRef(ConcatOps.data(), NumOps));
2762 }
2763 
2764 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2765  // Binary op widening for operations that can trap.
2766  unsigned Opcode = N->getOpcode();
2767  SDLoc dl(N);
2768  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2769  EVT WidenEltVT = WidenVT.getVectorElementType();
2770  EVT VT = WidenVT;
2771  unsigned NumElts = VT.getVectorNumElements();
2772  const SDNodeFlags Flags = N->getFlags();
2773  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2774  NumElts = NumElts / 2;
2775  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2776  }
2777 
2778  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2779  // Operation doesn't trap so just widen as normal.
2780  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2781  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2782  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2783  }
2784 
2785  // No legal vector version so unroll the vector operation and then widen.
2786  if (NumElts == 1)
2787  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2788 
2789  // Since the operation can trap, apply operation on the original vector.
2790  EVT MaxVT = VT;
2791  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2792  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2793  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2794 
2795  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2796  unsigned ConcatEnd = 0; // Current ConcatOps index.
2797  int Idx = 0; // Current Idx into input vectors.
2798 
2799  // NumElts := greatest legal vector size (at most WidenVT)
2800  // while (orig. vector has unhandled elements) {
2801  // take munches of size NumElts from the beginning and add to ConcatOps
2802  // NumElts := next smaller supported vector size or 1
2803  // }
2804  while (CurNumElts != 0) {
2805  while (CurNumElts >= NumElts) {
2806  SDValue EOp1 = DAG.getNode(
2807  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2808  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2809  SDValue EOp2 = DAG.getNode(
2810  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2811  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2812  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2813  Idx += NumElts;
2814  CurNumElts -= NumElts;
2815  }
2816  do {
2817  NumElts = NumElts / 2;
2818  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2819  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2820 
2821  if (NumElts == 1) {
2822  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2823  SDValue EOp1 = DAG.getNode(
2824  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2825  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2826  SDValue EOp2 = DAG.getNode(
2827  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2828  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2829  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2830  EOp1, EOp2, Flags);
2831  }
2832  CurNumElts = 0;
2833  }
2834  }
2835 
2836  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
2837 }
2838 
2839 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
2840  // StrictFP op widening for operations that can trap.
2841  unsigned NumOpers = N->getNumOperands();
2842  unsigned Opcode = N->getOpcode();
2843  SDLoc dl(N);
2844  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2845  EVT WidenEltVT = WidenVT.getVectorElementType();
2846  EVT VT = WidenVT;
2847  unsigned NumElts = VT.getVectorNumElements();
2848  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2849  NumElts = NumElts / 2;
2850  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2851  }
2852 
2853  // No legal vector version so unroll the vector operation and then widen.
2854  if (NumElts == 1)
2855  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2856 
2857  // Since the operation can trap, apply operation on the original vector.
2858  EVT MaxVT = VT;
2860  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2861 
2862  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2863  SmallVector<SDValue, 16> Chains;
2864  unsigned ConcatEnd = 0; // Current ConcatOps index.
2865  int Idx = 0; // Current Idx into input vectors.
2866 
2867  // The Chain is the first operand.
2868  InOps.push_back(N->getOperand(0));
2869 
2870  // Now process the remaining operands.
2871  for (unsigned i = 1; i < NumOpers; ++i) {
2872  SDValue Oper = N->getOperand(i);
2873 
2874  if (Oper.getValueType().isVector()) {
2875  assert(Oper.getValueType() == N->getValueType(0) &&
2876  "Invalid operand type to widen!");
2877  Oper = GetWidenedVector(Oper);
2878  }
2879 
2880  InOps.push_back(Oper);
2881  }
2882 
2883  // NumElts := greatest legal vector size (at most WidenVT)
2884  // while (orig. vector has unhandled elements) {
2885  // take munches of size NumElts from the beginning and add to ConcatOps
2886  // NumElts := next smaller supported vector size or 1
2887  // }
2888  while (CurNumElts != 0) {
2889  while (CurNumElts >= NumElts) {
2891 
2892  for (unsigned i = 0; i < NumOpers; ++i) {
2893  SDValue Op = InOps[i];
2894 
2895  if (Op.getValueType().isVector())
2896  Op = DAG.getNode(
2897  ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
2898  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2899 
2900  EOps.push_back(Op);
2901  }
2902 
2903  EVT OperVT[] = {VT, MVT::Other};
2904  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
2905  ConcatOps[ConcatEnd++] = Oper;
2906  Chains.push_back(Oper.getValue(1));
2907  Idx += NumElts;
2908  CurNumElts -= NumElts;
2909  }
2910  do {
2911  NumElts = NumElts / 2;
2912  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2913  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2914 
2915  if (NumElts == 1) {
2916  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2918 
2919  for (unsigned i = 0; i < NumOpers; ++i) {
2920  SDValue Op = InOps[i];
2921 
2922  if (Op.getValueType().isVector())
2923  Op = DAG.getNode(
2924  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
2925  DAG.getConstant(Idx, dl,
2926  TLI.getVectorIdxTy(DAG.getDataLayout())));
2927 
2928  EOps.push_back(Op);
2929  }
2930 
2931  EVT WidenVT[] = {WidenEltVT, MVT::Other};
2932  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
2933  ConcatOps[ConcatEnd++] = Oper;
2934  Chains.push_back(Oper.getValue(1));
2935  }
2936  CurNumElts = 0;
2937  }
2938  }
2939 
2940  // Build a factor node to remember all the Ops that have been created.
2941  SDValue NewChain;
2942  if (Chains.size() == 1)
2943  NewChain = Chains[0];
2944  else
2945  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
2946  ReplaceValueWith(SDValue(N, 1), NewChain);
2947 
2948  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
2949 }
2950 
2951 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
2952  SDLoc DL(N);
2953  EVT ResVT = N->getValueType(0);
2954  EVT OvVT = N->getValueType(1);
2955  EVT WideResVT, WideOvVT;
2956  SDValue WideLHS, WideRHS;
2957 
2958  // TODO: This might result in a widen/split loop.
2959  if (ResNo == 0) {
2960  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
2961  WideOvVT = EVT::getVectorVT(
2962  *DAG.getContext(), OvVT.getVectorElementType(),
2963  WideResVT.getVectorNumElements());
2964 
2965  WideLHS = GetWidenedVector(N->getOperand(0));
2966  WideRHS = GetWidenedVector(N->getOperand(1));
2967  } else {
2968  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
2969  WideResVT = EVT::getVectorVT(
2970  *DAG.getContext(), ResVT.getVectorElementType(),
2971  WideOvVT.getVectorNumElements());
2972 
2973  SDValue Zero = DAG.getConstant(
2974  0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
2975  WideLHS = DAG.getNode(
2976  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
2977  N->getOperand(0), Zero);
2978  WideRHS = DAG.getNode(
2979  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
2980  N->getOperand(1), Zero);
2981  }
2982 
2983  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
2984  SDNode *WideNode = DAG.getNode(
2985  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
2986 
2987  // Replace the other vector result not being explicitly widened here.
2988  unsigned OtherNo = 1 - ResNo;
2989  EVT OtherVT = N->getValueType(OtherNo);
2990  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
2991  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
2992  } else {
2993  SDValue Zero = DAG.getConstant(
2994  0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
2995  SDValue OtherVal = DAG.getNode(
2996  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
2997  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
2998  }
2999 
3000  return SDValue(WideNode, ResNo);
3001 }
3002 
3003 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3004  SDValue InOp = N->getOperand(0);
3005  SDLoc DL(N);
3006 
3007  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3008  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3009 
3010  EVT InVT = InOp.getValueType();
3011  EVT InEltVT = InVT.getVectorElementType();
3012  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
3013 
3014  unsigned Opcode = N->getOpcode();
3015  unsigned InVTNumElts = InVT.getVectorNumElements();
3016  const SDNodeFlags Flags = N->getFlags();
3017  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3018  InOp = GetWidenedVector(N->getOperand(0));
3019  InVT = InOp.getValueType();
3020  InVTNumElts = InVT.getVectorNumElements();
3021  if (InVTNumElts == WidenNumElts) {
3022  if (N->getNumOperands() == 1)
3023  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3024  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3025  }
3026  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3027  // If both input and result vector types are of same width, extend
3028  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3029  // accepts fewer elements in the result than in the input.
3030  if (Opcode == ISD::ANY_EXTEND)
3031  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3032  if (Opcode == ISD::SIGN_EXTEND)
3033  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3034  if (Opcode == ISD::ZERO_EXTEND)
3035  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3036  }
3037  }
3038 
3039  if (TLI.isTypeLegal(InWidenVT)) {
3040  // Because the result and the input are different vector types, widening
3041  // the result could create a legal type but widening the input might make
3042  // it an illegal type that might lead to repeatedly splitting the input
3043  // and then widening it. To avoid this, we widen the input only if
3044  // it results in a legal type.
3045  if (WidenNumElts % InVTNumElts == 0) {
3046  // Widen the input and call convert on the widened input vector.
3047  unsigned NumConcat = WidenNumElts/InVTNumElts;
3048  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3049  Ops[0] = InOp;
3050  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3051  if (N->getNumOperands() == 1)
3052  return DAG.getNode(Opcode, DL, WidenVT, InVec);
3053  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3054  }
3055 
3056  if (InVTNumElts % WidenNumElts == 0) {
3057  SDValue InVal = DAG.getNode(
3058  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3059  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3060  // Extract the input and convert the shorten input vector.
3061  if (N->getNumOperands() == 1)
3062  return DAG.getNode(Opcode, DL, WidenVT, InVal);
3063  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3064  }
3065  }
3066 
3067  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3068  EVT EltVT = WidenVT.getVectorElementType();
3069  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3070  // Use the original element count so we don't do more scalar opts than
3071  // necessary.
3072  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3073  for (unsigned i=0; i < MinElts; ++i) {
3074  SDValue Val = DAG.getNode(
3075  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3076  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3077  if (N->getNumOperands() == 1)
3078  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3079  else
3080  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3081  }
3082 
3083  return DAG.getBuildVector(WidenVT, DL, Ops);
3084 }
3085 
3086 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3087  unsigned Opcode = N->getOpcode();
3088  SDValue InOp = N->getOperand(0);
3089  SDLoc DL(N);
3090 
3091  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3092  EVT WidenSVT = WidenVT.getVectorElementType();
3093  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3094 
3095  EVT InVT = InOp.getValueType();
3096  EVT InSVT = InVT.getVectorElementType();
3097  unsigned InVTNumElts = InVT.getVectorNumElements();
3098 
3099  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3100  InOp = GetWidenedVector(InOp);
3101  InVT = InOp.getValueType();
3102  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3103  switch (Opcode) {
3107  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3108  }
3109  }
3110  }
3111 
3112  // Unroll, extend the scalars and rebuild the vector.
3114  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3115  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3116  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3117  switch (Opcode) {
3119  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3120  break;
3122  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3123  break;
3125  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3126  break;
3127  default:
3128  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3129  }
3130  Ops.push_back(Val);
3131  }
3132 
3133  while (Ops.size() != WidenNumElts)
3134  Ops.push_back(DAG.getUNDEF(WidenSVT));
3135 
3136  return DAG.getBuildVector(WidenVT, DL, Ops);
3137 }
3138 
3139 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3140  // If this is an FCOPYSIGN with same input types, we can treat it as a
3141  // normal (can trap) binary op.
3142  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3143  return WidenVecRes_BinaryCanTrap(N);
3144 
3145  // If the types are different, fall back to unrolling.
3146  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3147  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3148 }
3149 
3150 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3151  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3152  SDValue InOp = GetWidenedVector(N->getOperand(0));
3153  SDValue ShOp = N->getOperand(1);
3154  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3155 }
3156 
3157 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
3158  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3159  SDValue InOp = GetWidenedVector(N->getOperand(0));
3160  SDValue ShOp = N->getOperand(1);
3161 
3162  EVT ShVT = ShOp.getValueType();
3163  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
3164  ShOp = GetWidenedVector(ShOp);
3165  ShVT = ShOp.getValueType();
3166  }
3167  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
3168  ShVT.getVectorElementType(),
3169  WidenVT.getVectorNumElements());
3170  if (ShVT != ShWidenVT)
3171  ShOp = ModifyToType(ShOp, ShWidenVT);
3172 
3173  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3174 }
3175 
3176 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3177  // Unary op widening.
3178  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3179  SDValue InOp = GetWidenedVector(N->getOperand(0));
3180  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3181 }
3182 
3183 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3184  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3185  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3186  cast<VTSDNode>(N->getOperand(1))->getVT()
3187  .getVectorElementType(),
3188  WidenVT.getVectorNumElements());
3189  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3190  return DAG.getNode(N->getOpcode(), SDLoc(N),
3191  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3192 }
3193 
3194 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3195  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3196  return GetWidenedVector(WidenVec);
3197 }
3198 
3199 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3200  SDValue InOp = N->getOperand(0);
3201  EVT InVT = InOp.getValueType();
3202  EVT VT = N->getValueType(0);
3203  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3204  SDLoc dl(N);
3205 
3206  switch (getTypeAction(InVT)) {
3208  break;
3210  // If the incoming type is a vector that is being promoted, then
3211  // we know that the elements are arranged differently and that we
3212  // must perform the conversion using a stack slot.
3213  if (InVT.isVector())
3214  break;
3215 
3216  // If the InOp is promoted to the same size, convert it. Otherwise,
3217  // fall out of the switch and widen the promoted input.
3218  InOp = GetPromotedInteger(InOp);
3219  InVT = InOp.getValueType();
3220  if (WidenVT.bitsEq(InVT))
3221  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3222  break;
3229  break;
3231  // If the InOp is widened to the same size, convert it. Otherwise, fall
3232  // out of the switch and widen the widened input.
3233  InOp = GetWidenedVector(InOp);
3234  InVT = InOp.getValueType();
3235  if (WidenVT.bitsEq(InVT))
3236  // The input widens to the same size. Convert to the widen value.
3237  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3238  break;
3239  }
3240 
3241  unsigned WidenSize = WidenVT.getSizeInBits();
3242  unsigned InSize = InVT.getSizeInBits();
3243  // x86mmx is not an acceptable vector element type, so don't try.
3244  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3245  // Determine new input vector type. The new input vector type will use
3246  // the same element type (if its a vector) or use the input type as a
3247  // vector. It is the same size as the type to widen to.
3248  EVT NewInVT;
3249  unsigned NewNumElts = WidenSize / InSize;
3250  if (InVT.isVector()) {
3251  EVT InEltVT = InVT.getVectorElementType();
3252  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3253  WidenSize / InEltVT.getSizeInBits());
3254  } else {
3255  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3256  }
3257 
3258  if (TLI.isTypeLegal(NewInVT)) {
3259  SDValue NewVec;
3260  if (InVT.isVector()) {
3261  // Because the result and the input are different vector types, widening
3262  // the result could create a legal type but widening the input might make
3263  // it an illegal type that might lead to repeatedly splitting the input
3264  // and then widening it. To avoid this, we widen the input only if
3265  // it results in a legal type.
3266  SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3267  Ops[0] = InOp;
3268 
3269  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3270  } else {
3271  NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3272  }
3273  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3274  }
3275  }
3276 
3277  return CreateStackStoreLoad(InOp, WidenVT);
3278 }
3279 
3280 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3281  SDLoc dl(N);
3282  // Build a vector with undefined for the new nodes.
3283  EVT VT = N->getValueType(0);
3284 
3285  // Integer BUILD_VECTOR operands may be larger than the node's vector element
3286  // type. The UNDEFs need to have the same type as the existing operands.
3287  EVT EltVT = N->getOperand(0).getValueType();
3288  unsigned NumElts = VT.getVectorNumElements();
3289 
3290  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3291  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3292 
3293  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3294  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3295  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3296 
3297  return DAG.getBuildVector(WidenVT, dl, NewOps);
3298 }
3299 
3300 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3301  EVT InVT = N->getOperand(0).getValueType();
3302  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3303  SDLoc dl(N);
3304  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3305  unsigned NumInElts = InVT.getVectorNumElements();
3306  unsigned NumOperands = N->getNumOperands();
3307 
3308  bool InputWidened = false; // Indicates we need to widen the input.
3309  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3310  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
3311  // Add undef vectors to widen to correct length.
3312  unsigned NumConcat = WidenVT.getVectorNumElements() /
3313  InVT.getVectorNumElements();
3314  SDValue UndefVal = DAG.getUNDEF(InVT);
3315  SmallVector<SDValue, 16> Ops(NumConcat);
3316  for (unsigned i=0; i < NumOperands; ++i)
3317  Ops[i] = N->getOperand(i);
3318  for (unsigned i = NumOperands; i != NumConcat; ++i)
3319  Ops[i] = UndefVal;
3320  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3321  }
3322  } else {
3323  InputWidened = true;
3324  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3325  // The inputs and the result are widen to the same value.
3326  unsigned i;
3327  for (i=1; i < NumOperands; ++i)
3328  if (!N->getOperand(i).isUndef())
3329  break;
3330 
3331  if (i == NumOperands)
3332  // Everything but the first operand is an UNDEF so just return the
3333  // widened first operand.
3334  return GetWidenedVector(N->getOperand(0));
3335 
3336  if (NumOperands == 2) {
3337  // Replace concat of two operands with a shuffle.
3338  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3339  for (unsigned i = 0; i < NumInElts; ++i) {
3340  MaskOps[i] = i;
3341  MaskOps[i + NumInElts] = i + WidenNumElts;
3342  }
3343  return DAG.getVectorShuffle(WidenVT, dl,
3344  GetWidenedVector(N->getOperand(0)),
3345  GetWidenedVector(N->getOperand(1)),
3346  MaskOps);
3347  }
3348  }
3349  }
3350 
3351  // Fall back to use extracts and build vector.
3352  EVT EltVT = WidenVT.getVectorElementType();
3353  SmallVector<SDValue, 16> Ops(WidenNumElts);
3354  unsigned Idx = 0;
3355  for (unsigned i=0; i < NumOperands; ++i) {
3356  SDValue InOp = N->getOperand(i);
3357  if (InputWidened)
3358  InOp = GetWidenedVector(InOp);
3359  for (unsigned j=0; j < NumInElts; ++j)
3360  Ops[Idx++] = DAG.getNode(
3361  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3362  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3363  }
3364  SDValue UndefVal = DAG.getUNDEF(EltVT);
3365  for (; Idx < WidenNumElts; ++Idx)
3366  Ops[Idx] = UndefVal;
3367  return DAG.getBuildVector(WidenVT, dl, Ops);
3368 }
3369 
3370 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3371  EVT VT = N->getValueType(0);
3372  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3373  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3374  SDValue InOp = N->getOperand(0);
3375  SDValue Idx = N->getOperand(1);
3376  SDLoc dl(N);
3377 
3378  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3379  InOp = GetWidenedVector(InOp);
3380 
3381  EVT InVT = InOp.getValueType();
3382 
3383  // Check if we can just return the input vector after widening.
3384  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3385  if (IdxVal == 0 && InVT == WidenVT)
3386  return InOp;
3387 
3388  // Check if we can extract from the vector.
3389  unsigned InNumElts = InVT.getVectorNumElements();
3390  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3391  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3392 
3393  // We could try widening the input to the right length but for now, extract
3394  // the original elements, fill the rest with undefs and build a vector.
3395  SmallVector<SDValue, 16> Ops(WidenNumElts);
3396  EVT EltVT = VT.getVectorElementType();
3397  unsigned NumElts = VT.getVectorNumElements();
3398  unsigned i;
3399  for (i=0; i < NumElts; ++i)
3400  Ops[i] =
3401  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3402  DAG.getConstant(IdxVal + i, dl,
3403  TLI.getVectorIdxTy(DAG.getDataLayout())));
3404 
3405  SDValue UndefVal = DAG.getUNDEF(EltVT);
3406  for (; i < WidenNumElts; ++i)
3407  Ops[i] = UndefVal;
3408  return DAG.getBuildVector(WidenVT, dl, Ops);
3409 }
3410 
3411 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3412  SDValue InOp = GetWidenedVector(N->getOperand(0));
3413  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3414  InOp.getValueType(), InOp,
3415  N->getOperand(1), N->getOperand(2));
3416 }
3417 
3418 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3419  LoadSDNode *LD = cast<LoadSDNode>(N);
3421 
3422  SDValue Result;
3423  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3424  if (ExtType != ISD::NON_EXTLOAD)
3425  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3426  else
3427  Result = GenWidenVectorLoads(LdChain, LD);
3428 
3429  // If we generate a single load, we can use that for the chain. Otherwise,
3430  // build a factor node to remember the multiple loads are independent and
3431  // chain to that.
3432  SDValue NewChain;
3433  if (LdChain.size() == 1)
3434  NewChain = LdChain[0];
3435  else
3436  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3437 
3438  // Modified the chain - switch anything that used the old chain to use
3439  // the new one.
3440  ReplaceValueWith(SDValue(N, 1), NewChain);
3441 
3442  return Result;
3443 }
3444 
3445 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3446 
3447  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3448  SDValue Mask = N->getMask();
3449  EVT MaskVT = Mask.getValueType();
3450  SDValue PassThru = GetWidenedVector(N->getPassThru());
3452  SDLoc dl(N);
3453 
3454  // The mask should be widened as well
3455  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3456  MaskVT.getVectorElementType(),
3457  WidenVT.getVectorNumElements());
3458  Mask = ModifyToType(Mask, WideMaskVT, true);
3459 
3460  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3461  Mask, PassThru, N->getMemoryVT(),
3462  N->getMemOperand(), ExtType,
3463  N->isExpandingLoad());
3464  // Legalize the chain result - switch anything that used the old chain to
3465  // use the new one.
3466  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3467  return Res;
3468 }
3469 
3470 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3471 
3472  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3473  SDValue Mask = N->getMask();
3474  EVT MaskVT = Mask.getValueType();
3475  SDValue PassThru = GetWidenedVector(N->getPassThru());
3476  SDValue Scale = N->getScale();
3477  unsigned NumElts = WideVT.getVectorNumElements();
3478  SDLoc dl(N);
3479 
3480  // The mask should be widened as well
3481  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3482  MaskVT.getVectorElementType(),
3483  WideVT.getVectorNumElements());
3484  Mask = ModifyToType(Mask, WideMaskVT, true);
3485 
3486  // Widen the Index operand
3487  SDValue Index = N->getIndex();
3488  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3489  Index.getValueType().getScalarType(),
3490  NumElts);
3491  Index = ModifyToType(Index, WideIndexVT);
3492  SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3493  Scale };
3494  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3495  N->getMemoryVT(), dl, Ops,
3496  N->getMemOperand());
3497 
3498  // Legalize the chain result - switch anything that used the old chain to
3499  // use the new one.
3500  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3501  return Res;
3502 }
3503 
3504 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3505  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3506  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3507  WidenVT, N->getOperand(0));
3508 }
3509 
3510 // Return true if this is a node that could have two SETCCs as operands.
3511 static inline bool isLogicalMaskOp(unsigned Opcode) {
3512  switch (Opcode) {
3513  case ISD::AND:
3514  case ISD::OR:
3515  case ISD::XOR:
3516  return true;
3517  }
3518  return false;
3519 }
3520 
3521 // This is used just for the assert in convertMask(). Check that this either
3522 // a SETCC or a previously handled SETCC by convertMask().
3523 #ifndef NDEBUG
3524 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3525  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3526  N = N.getOperand(0);
3527  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3528  for (unsigned i = 1; i < N->getNumOperands(); ++i)
3529  if (!N->getOperand(i)->isUndef())
3530  return false;
3531  N = N.getOperand(0);
3532  }
3533 
3534  if (N.getOpcode() == ISD::TRUNCATE)
3535  N = N.getOperand(0);
3536  else if (N.getOpcode() == ISD::SIGN_EXTEND)
3537  N = N.getOperand(0);
3538 
3539  if (isLogicalMaskOp(N.getOpcode()))
3540  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3542 
3543  return (N.getOpcode() == ISD::SETCC ||
3545 }
3546 #endif
3547 
3548 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3549 // to ToMaskVT if needed with vector extension or truncation.
3550 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3551  EVT ToMaskVT) {
3552  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3553  // FIXME: This code seems to be too restrictive, we might consider
3554  // generalizing it or dropping it.
3555  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3556 
3557  // Make a new Mask node, with a legal result VT.
3559  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3560  Ops.push_back(InMask->getOperand(i));
3561  SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3562 
3563  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3564  // extend or truncate is needed.
3565  LLVMContext &Ctx = *DAG.getContext();
3566  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3567  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3568  if (MaskScalarBits < ToMaskScalBits) {
3569  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3570  MaskVT.getVectorNumElements());
3571  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3572  } else if (MaskScalarBits > ToMaskScalBits) {
3573  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3574  MaskVT.getVectorNumElements());
3575  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3576  }
3577 
3579  ToMaskVT.getScalarSizeInBits() &&
3580  "Mask should have the right element size by now.");
3581 
3582  // Adjust Mask to the right number of elements.
3583  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3584  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3585  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3586  SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3587  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3588  ZeroIdx);
3589  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3590  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3591  EVT SubVT = Mask->getValueType(0);
3592  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3593  SubOps[0] = Mask;
3594  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3595  }
3596 
3597  assert((Mask->getValueType(0) == ToMaskVT) &&
3598  "A mask of ToMaskVT should have been produced by now.");
3599 
3600  return Mask;
3601 }
3602 
3603 // This method tries to handle VSELECT and its mask by legalizing operands
3604 // (which may require widening) and if needed adjusting the mask vector type
3605 // to match that of the VSELECT. Without it, many cases end up with
3606 // scalarization of the SETCC, with many unnecessary instructions.
3607 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3608  LLVMContext &Ctx = *DAG.getContext();
3609  SDValue Cond = N->getOperand(0);
3610 
3611  if (N->getOpcode() != ISD::VSELECT)
3612  return SDValue();
3613 
3614  if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3615  return SDValue();
3616 
3617  // If this is a splitted VSELECT that was previously already handled, do
3618  // nothing.
3619  EVT CondVT = Cond->getValueType(0);
3620  if (CondVT.getScalarSizeInBits() != 1)
3621  return SDValue();
3622 
3623  EVT VSelVT = N->getValueType(0);
3624  // Only handle vector types which are a power of 2.
3625  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3626  return SDValue();
3627 
3628  // Don't touch if this will be scalarized.
3629  EVT FinalVT = VSelVT;
3630  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3631  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3632 
3633  if (FinalVT.getVectorNumElements() == 1)
3634  return SDValue();
3635 
3636  // If there is support for an i1 vector mask, don't touch.
3637  if (Cond.getOpcode() == ISD::SETCC) {
3638  EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3639  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3640  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3641  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3642  if (SetCCResVT.getScalarSizeInBits() == 1)
3643  return SDValue();
3644  } else if (CondVT.getScalarType() == MVT::i1) {
3645  // If there is support for an i1 vector mask (or only scalar i1 conditions),
3646  // don't touch.
3647  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3648  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3649 
3650  if (CondVT.getScalarType() == MVT::i1)
3651  return SDValue();
3652  }
3653 
3654  // Get the VT and operands for VSELECT, and widen if needed.
3655  SDValue VSelOp1 = N->getOperand(1);
3656  SDValue VSelOp2 = N->getOperand(2);
3657  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3658  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3659  VSelOp1 = GetWidenedVector(VSelOp1);
3660  VSelOp2 = GetWidenedVector(VSelOp2);
3661  }
3662 
3663  // The mask of the VSELECT should have integer elements.
3664  EVT ToMaskVT = VSelVT;
3665  if (!ToMaskVT.getScalarType().isInteger())
3666  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3667 
3668  SDValue Mask;
3669  if (Cond->getOpcode() == ISD::SETCC) {
3670  EVT MaskVT = getSetCCResultType(Cond.getOperand(0).getValueType());
3671  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3672  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3673  Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3674  Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3675  // Cond is (AND/OR/XOR (SETCC, SETCC))
3676  SDValue SETCC0 = Cond->getOperand(0);
3677  SDValue SETCC1 = Cond->getOperand(1);
3678  EVT VT0 = getSetCCResultType(SETCC0.getOperand(0).getValueType());
3679  EVT VT1 = getSetCCResultType(SETCC1.getOperand(0).getValueType());
3680  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3681  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3682  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3683  EVT MaskVT;
3684  // If the two SETCCs have different VTs, either extend/truncate one of
3685  // them to the other "towards" ToMaskVT, or truncate one and extend the
3686  // other to ToMaskVT.
3687  if (ScalarBits0 != ScalarBits1) {
3688  EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3689  EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3690  if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3691  MaskVT = WideVT;
3692  else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3693  MaskVT = NarrowVT;
3694  else
3695  MaskVT = ToMaskVT;
3696  } else
3697  // If the two SETCCs have the same VT, don't change it.
3698  MaskVT = VT0;
3699 
3700  // Make new SETCCs and logical nodes.
3701  SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3702  SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3703  Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3704 
3705  // Convert the logical op for VSELECT if needed.
3706  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3707  } else
3708  return SDValue();
3709 
3710  return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3711 }
3712 
3713 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3714  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3715  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3716 
3717  SDValue Cond1 = N->getOperand(0);
3718  EVT CondVT = Cond1.getValueType();
3719  if (CondVT.isVector()) {
3720  if (SDValue Res = WidenVSELECTAndMask(N))
3721  return Res;
3722 
3723  EVT CondEltVT = CondVT.getVectorElementType();
3724  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3725  CondEltVT, WidenNumElts);
3726  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3727  Cond1 = GetWidenedVector(Cond1);
3728 
3729  // If we have to split the condition there is no point in widening the
3730  // select. This would result in an cycle of widening the select ->
3731  // widening the condition operand -> splitting the condition operand ->
3732  // splitting the select -> widening the select. Instead split this select
3733  // further and widen the resulting type.
3734  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3735  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3736  SDValue Res = ModifyToType(SplitSelect, WidenVT);
3737  return Res;
3738  }
3739 
3740  if (Cond1.getValueType() != CondWidenVT)
3741  Cond1 = ModifyToType(Cond1, CondWidenVT);
3742  }
3743 
3744  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3745  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3746  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3747  return DAG.getNode(N->getOpcode(), SDLoc(N),
3748  WidenVT, Cond1, InOp1, InOp2);
3749 }
3750 
3751 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3752  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3753  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3754  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3755  InOp1.getValueType(), N->getOperand(0),
3756  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3757 }
3758 
3759 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3760  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3761  return DAG.getUNDEF(WidenVT);
3762 }
3763 
3764 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3765  EVT VT = N->getValueType(0);
3766  SDLoc dl(N);
3767 
3768  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3769  unsigned NumElts = VT.getVectorNumElements();
3770  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3771 
3772  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3773  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3774 
3775  // Adjust mask based on new input vector length.
3776  SmallVector<int, 16> NewMask;
3777  for (unsigned i = 0; i != NumElts; ++i) {
3778  int Idx = N->getMaskElt(i);
3779  if (Idx < (int)NumElts)
3780  NewMask.push_back(Idx);
3781  else
3782  NewMask.push_back(Idx - NumElts + WidenNumElts);
3783  }
3784  for (unsigned i = NumElts; i != WidenNumElts; ++i)
3785  NewMask.push_back(-1);
3786  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
3787 }
3788 
3789 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
3790  assert(N->getValueType(0).isVector() &&
3791  N->getOperand(0).getValueType().isVector() &&
3792  "Operands must be vectors");
3793  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3794  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3795 
3796  SDValue InOp1 = N->getOperand(0);
3797  EVT InVT = InOp1.getValueType();
3798  assert(InVT.isVector() && "can not widen non-vector type");
3799  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
3800  InVT.getVectorElementType(), WidenNumElts);
3801 
3802  // The input and output types often differ here, and it could be that while
3803  // we'd prefer to widen the result type, the input operands have been split.
3804  // In this case, we also need to split the result of this node as well.
3805  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
3806  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
3807  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
3808  return Res;
3809  }
3810 
3811  InOp1 = GetWidenedVector(InOp1);
3812  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3813 
3814  // Assume that the input and output will be widen appropriately. If not,
3815  // we will have to unroll it at some point.
3816  assert(InOp1.getValueType() == WidenInVT &&
3817  InOp2.getValueType() == WidenInVT &&
3818  "Input not widened to expected type!");
3819  (void)WidenInVT;
3820  return DAG.getNode(ISD::SETCC, SDLoc(N),
3821  WidenVT, InOp1, InOp2, N->getOperand(2));
3822 }
3823 
3824 
3825 //===----------------------------------------------------------------------===//
3826 // Widen Vector Operand
3827 //===----------------------------------------------------------------------===//
3828 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
3829  LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
3830  dbgs() << "\n");
3831  SDValue Res = SDValue();
3832 
3833  // See if the target wants to custom widen this node.
3834  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3835  return false;
3836 
3837  switch (N->getOpcode()) {
3838  default:
3839 #ifndef NDEBUG
3840  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
3841  N->dump(&DAG);
3842  dbgs() << "\n";
3843 #endif
3844  llvm_unreachable("Do not know how to widen this operator's operand!");
3845 
3846  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
3847  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
3848  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
3849  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
3850  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
3851  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
3852  case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
3853  case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
3854  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
3855  case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
3856 
3857  case ISD::ANY_EXTEND:
3858  case ISD::SIGN_EXTEND:
3859  case ISD::ZERO_EXTEND:
3860  Res = WidenVecOp_EXTEND(N);
3861  break;
3862 
3863  case ISD::FP_EXTEND:
3864  case ISD::FP_TO_SINT:
3865  case ISD::FP_TO_UINT:
3866  case ISD::SINT_TO_FP:
3867  case ISD::UINT_TO_FP:
3868  case ISD::TRUNCATE:
3869  Res = WidenVecOp_Convert(N);
3870  break;
3871  }
3872 
3873  // If Res is null, the sub-method took care of registering the result.
3874  if (!Res.getNode()) return false;
3875 
3876  // If the result is N, the sub-method updated N in place. Tell the legalizer
3877  // core about this.
3878  if (Res.getNode() == N)
3879  return true;
3880 
3881 
3882  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3883  "Invalid operand expansion");
3884 
3885  ReplaceValueWith(SDValue(N, 0), Res);
3886  return false;
3887 }
3888 
3889 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
3890  SDLoc DL(N);
3891  EVT VT = N->getValueType(0);
3892 
3893  SDValue InOp = N->getOperand(0);
3894  assert(getTypeAction(InOp.getValueType()) ==
3896  "Unexpected type action");
3897  InOp = GetWidenedVector(InOp);
3900  "Input wasn't widened!");
3901 
3902  // We may need to further widen the operand until it has the same total
3903  // vector size as the result.
3904  EVT InVT = InOp.getValueType();
3905  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
3906  EVT InEltVT = InVT.getVectorElementType();
3907  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
3908  EVT FixedVT = (MVT::SimpleValueType)i;
3909  EVT FixedEltVT = FixedVT.getVectorElementType();
3910  if (TLI.isTypeLegal(FixedVT) &&
3911  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
3912  FixedEltVT == InEltVT) {
3913  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
3914  "Not enough elements in the fixed type for the operand!");
3915  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
3916  "We can't have the same type as we started with!");
3917  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
3918  InOp = DAG.getNode(
3919  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
3920  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3921  else
3922  InOp = DAG.getNode(
3923  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
3924  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3925  break;
3926  }
3927  }
3928  InVT = InOp.getValueType();
3929  if (InVT.getSizeInBits() != VT.getSizeInBits())
3930  // We couldn't find a legal vector type that was a widening of the input
3931  // and could be extended in-register to the result type, so we have to
3932  // scalarize.
3933  return WidenVecOp_Convert(N);
3934  }
3935 
3936  // Use special DAG nodes to represent the operation of extending the
3937  // low lanes.
3938  switch (N->getOpcode()) {
3939  default:
3940  llvm_unreachable("Extend legalization on extend operation!");
3941  case ISD::ANY_EXTEND:
3942  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
3943  case ISD::SIGN_EXTEND:
3944  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
3945  case ISD::ZERO_EXTEND:
3946  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
3947  }
3948 }
3949 
3950 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
3951  // The result (and first input) is legal, but the second input is illegal.
3952  // We can't do much to fix that, so just unroll and let the extracts off of
3953  // the second input be widened as needed later.
3954  return DAG.UnrollVectorOp(N);
3955 }
3956 
3957 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
3958  // Since the result is legal and the input is illegal.
3959  EVT VT = N->getValueType(0);
3960  EVT EltVT = VT.getVectorElementType();
3961  SDLoc dl(N);
3962  unsigned NumElts = VT.getVectorNumElements();
3963  SDValue InOp = N->getOperand(0);
3964  assert(getTypeAction(InOp.getValueType()) ==
3966  "Unexpected type action");
3967  InOp = GetWidenedVector(InOp);
3968  EVT InVT = InOp.getValueType();
3969  unsigned Opcode = N->getOpcode();
3970 
3971  // See if a widened result type would be legal, if so widen the node.
3972  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3973  InVT.getVectorNumElements());
3974  if (TLI.isTypeLegal(WideVT)) {
3975  SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp);
3976  return DAG.getNode(
3977  ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
3978  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3979  }
3980 
3981  EVT InEltVT = InVT.getVectorElementType();
3982 
3983  // Unroll the convert into some scalar code and create a nasty build vector.
3984  SmallVector<SDValue, 16> Ops(NumElts);
3985  for (unsigned i=0; i < NumElts; ++i)
3986  Ops[i] = DAG.getNode(
3987  Opcode, dl, EltVT,
3988  DAG.getNode(
3989  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
3990  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3991 
3992  return DAG.getBuildVector(VT, dl, Ops);
3993 }
3994 
3995 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
3996  EVT VT = N->getValueType(0);
3997  SDValue InOp = GetWidenedVector(N->getOperand(0));
3998  EVT InWidenVT = InOp.getValueType();
3999  SDLoc dl(N);
4000 
4001  // Check if we can convert between two legal vector types and extract.
4002  unsigned InWidenSize = InWidenVT.getSizeInBits();
4003  unsigned Size = VT.getSizeInBits();
4004  // x86mmx is not an acceptable vector element type, so don't try.
4005  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
4006  unsigned NewNumElts = InWidenSize / Size;
4007  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4008  if (TLI.isTypeLegal(NewVT)) {
4009  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4010  return DAG.getNode(
4011  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4012  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4013  }
4014  }
4015 
4016  return CreateStackStoreLoad(InOp, VT);
4017 }
4018 
4019 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4020  EVT VT = N->getValueType(0);
4021  EVT EltVT = VT.getVectorElementType();
4022  EVT InVT = N->getOperand(0).getValueType();
4023  SDLoc dl(N);
4024 
4025  // If the widen width for this operand is the same as the width of the concat
4026  // and all but the first operand is undef, just use the widened operand.
4027  unsigned NumOperands = N->getNumOperands();
4028  if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4029  unsigned i;
4030  for (i = 1; i < NumOperands; ++i)
4031  if (!N->getOperand(i).isUndef())
4032  break;
4033 
4034  if (i == NumOperands)
4035  return GetWidenedVector(N->getOperand(0));
4036  }
4037 
4038  // Otherwise, fall back to a nasty build vector.
4039  unsigned NumElts = VT.getVectorNumElements();
4040  SmallVector<SDValue, 16> Ops(NumElts);
4041 
4042  unsigned NumInElts = InVT.getVectorNumElements();
4043 
4044  unsigned Idx = 0;
4045  for (unsigned i=0; i < NumOperands; ++i) {
4046  SDValue InOp = N->getOperand(i);
4047  assert(getTypeAction(InOp.getValueType()) ==
4049  "Unexpected type action");
4050  InOp = GetWidenedVector(InOp);
4051  for (unsigned j=0; j < NumInElts; ++j)
4052  Ops[Idx++] = DAG.getNode(
4053  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4054  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4055  }
4056  return DAG.getBuildVector(VT, dl, Ops);
4057 }
4058 
4059 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4060  SDValue InOp = GetWidenedVector(N->getOperand(0));
4061  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4062  N->getValueType(0), InOp, N->getOperand(1));
4063 }
4064 
4065 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4066  SDValue InOp = GetWidenedVector(N->getOperand(0));
4067  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4068  N->getValueType(0), InOp, N->getOperand(1));
4069 }
4070 
4071 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4072  // We have to widen the value, but we want only to store the original
4073  // vector type.
4074  StoreSDNode *ST = cast<StoreSDNode>(N);
4075 
4076  if (!ST->getMemoryVT().getScalarType().isByteSized())
4077  return TLI.scalarizeVectorStore(ST, DAG);
4078 
4079  SmallVector<SDValue, 16> StChain;
4080  if (ST->isTruncatingStore())
4081  GenWidenVectorTruncStores(StChain, ST);
4082  else
4083  GenWidenVectorStores(StChain, ST);
4084 
4085  if (StChain.size() == 1)
4086  return StChain[0];
4087  else
4088  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4089 }
4090 
4091 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4092  assert((OpNo == 1 || OpNo == 3) &&
4093  "Can widen only data or mask operand of mstore");
4094  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4095  SDValue Mask = MST->getMask();
4096  EVT MaskVT = Mask.getValueType();
4097  SDValue StVal = MST->getValue();
4098  SDLoc dl(N);
4099 
4100  if (OpNo == 1) {
4101  // Widen the value.
4102  StVal = GetWidenedVector(StVal);
4103 
4104  // The mask should be widened as well.
4105  EVT WideVT = StVal.getValueType();
4106  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4107  MaskVT.getVectorElementType(),
4108  WideVT.getVectorNumElements());
4109  Mask = ModifyToType(Mask, WideMaskVT, true);
4110  } else {
4111  // Widen the mask.
4112  EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4113  Mask = ModifyToType(Mask, WideMaskVT, true);
4114 
4115  EVT ValueVT = StVal.getValueType();
4116  EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4117  ValueVT.getVectorElementType(),
4118  WideMaskVT.getVectorNumElements());
4119  StVal = ModifyToType(StVal, WideVT);
4120  }
4121 
4122  assert(Mask.getValueType().getVectorNumElements() ==
4123  StVal.getValueType().getVectorNumElements() &&
4124  "Mask and data vectors should have the same number of elements");
4125  return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4126  Mask, MST->getMemoryVT(), MST->getMemOperand(),
4127  false, MST->isCompressingStore());
4128 }
4129 
4130 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4131  assert(OpNo == 4 && "Can widen only the index of mgather");
4132  auto *MG = cast<MaskedGatherSDNode>(N);
4133  SDValue DataOp = MG->getPassThru();
4134  SDValue Mask = MG->getMask();
4135  SDValue Scale = MG->getScale();
4136 
4137  // Just widen the index. It's allowed to have extra elements.
4138  SDValue Index = GetWidenedVector(MG->getIndex());
4139 
4140  SDLoc dl(N);
4141  SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4142  Scale};
4143  SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4144  MG->getMemOperand());
4145  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4146  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4147  return SDValue();
4148 }
4149 
4150 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4151  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4152  SDValue DataOp = MSC->getValue();
4153  SDValue Mask = MSC->getMask();
4154  SDValue Index = MSC->getIndex();
4155  SDValue Scale = MSC->getScale();
4156 
4157  unsigned NumElts;
4158  if (OpNo == 1) {
4159  DataOp = GetWidenedVector(DataOp);
4160  NumElts = DataOp.getValueType().getVectorNumElements();
4161 
4162  // Widen index.
4163  EVT IndexVT = Index.getValueType();
4164  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4165  IndexVT.getVectorElementType(), NumElts);
4166  Index = ModifyToType(Index, WideIndexVT);
4167 
4168  // The mask should be widened as well.
4169  EVT MaskVT = Mask.getValueType();
4170  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4171  MaskVT.getVectorElementType(), NumElts);
4172  Mask = ModifyToType(Mask, WideMaskVT, true);
4173  } else if (OpNo == 4) {
4174  // Just widen the index. It's allowed to have extra elements.
4175  Index = GetWidenedVector(Index);
4176  } else
4177  llvm_unreachable("Can't widen this operand of mscatter");
4178 
4179  SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4180  Scale};
4181  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4182  MSC->getMemoryVT(), SDLoc(N), Ops,
4183  MSC->getMemOperand());
4184 }
4185 
4186 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4187  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4188  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4189  SDLoc dl(N);
4190  EVT VT = N->getValueType(0);
4191 
4192  // WARNING: In this code we widen the compare instruction with garbage.
4193  // This garbage may contain denormal floats which may be slow. Is this a real
4194  // concern ? Should we zero the unused lanes if this is a float compare ?
4195 
4196  // Get a new SETCC node to compare the newly widened operands.
4197  // Only some of the compared elements are legal.
4198  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
4199  InOp0.getValueType());
4200  // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
4201  if (VT.getScalarType() == MVT::i1)
4202  SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4203  SVT.getVectorNumElements());
4204 
4205  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4206  SVT, InOp0, InOp1, N->getOperand(2));
4207 
4208  // Extract the needed results from the result vector.
4209  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
4210  SVT.getVectorElementType(),
4211  VT.getVectorNumElements());
4212  SDValue CC = DAG.getNode(
4213  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
4214  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4215 
4216  return PromoteTargetBoolean(CC, VT);
4217 }
4218 
4219 
4220 //===----------------------------------------------------------------------===//
4221 // Vector Widening Utilities
4222 //===----------------------------------------------------------------------===//
4223 
4224 // Utility function to find the type to chop up a widen vector for load/store
4225 // TLI: Target lowering used to determine legal types.
4226 // Width: Width left need to load/store.
4227 // WidenVT: The widen vector type to load to/store from
4228 // Align: If 0, don't allow use of a wider type
4229 // WidenEx: If Align is not 0, the amount additional we can load/store from.
4230 
4231 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4232  unsigned Width, EVT WidenVT,
4233  unsigned Align = 0, unsigned WidenEx = 0) {
4234  EVT WidenEltVT = WidenVT.getVectorElementType();
4235  unsigned WidenWidth = WidenVT.getSizeInBits();
4236  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4237  unsigned AlignInBits = Align*8;
4238 
4239  // If we have one element to load/store, return it.
4240  EVT RetVT = WidenEltVT;
4241  if (Width == WidenEltWidth)
4242  return RetVT;
4243 
4244  // See if there is larger legal integer than the element type to load/store.
4245  unsigned VT;
4246  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4247  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
4248  EVT MemVT((MVT::SimpleValueType) VT);
4249  unsigned MemVTWidth = MemVT.getSizeInBits();
4250  if (MemVT.getSizeInBits() <= WidenEltWidth)
4251  break;
4252  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4253  if ((Action == TargetLowering::TypeLegal ||
4255  (WidenWidth % MemVTWidth) == 0 &&
4256  isPowerOf2_32(WidenWidth / MemVTWidth) &&
4257  (MemVTWidth <= Width ||
4258  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4259  RetVT = MemVT;
4260  break;
4261  }
4262  }
4263 
4264  // See if there is a larger vector type to load/store that has the same vector
4265  // element type and is evenly divisible with the WidenVT.
4266  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4267  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
4268  EVT MemVT = (MVT::SimpleValueType) VT;
4269  unsigned MemVTWidth = MemVT.getSizeInBits();
4270  if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
4271  (WidenWidth % MemVTWidth) == 0 &&
4272  isPowerOf2_32(WidenWidth / MemVTWidth) &&
4273  (MemVTWidth <= Width ||
4274  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4275  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
4276  return MemVT;
4277  }
4278  }
4279 
4280  return RetVT;
4281 }
4282 
4283 // Builds a vector type from scalar loads
4284 // VecTy: Resulting Vector type
4285 // LDOps: Load operators to build a vector type
4286 // [Start,End) the list of loads to use.
4288  SmallVectorImpl<SDValue> &LdOps,
4289  unsigned Start, unsigned End) {
4290  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4291  SDLoc dl(LdOps[Start]);
4292  EVT LdTy = LdOps[Start].getValueType();
4293  unsigned Width = VecTy.getSizeInBits();
4294  unsigned NumElts = Width / LdTy.getSizeInBits();
4295  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4296 
4297  unsigned Idx = 1;
4298  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4299 
4300  for (unsigned i = Start + 1; i != End; ++i) {
4301  EVT NewLdTy = LdOps[i].getValueType();
4302  if (NewLdTy != LdTy) {
4303  NumElts = Width / NewLdTy.getSizeInBits();
4304  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4305  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4306  // Readjust position and vector position based on new load type.
4307  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4308  LdTy = NewLdTy;
4309  }
4310  VecOp = DAG.getNode(
4311  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4312  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4313  }
4314  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4315 }
4316 
4317 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4318  LoadSDNode *LD) {
4319  // The strategy assumes that we can efficiently load power-of-two widths.
4320  // The routine chops the vector into the largest vector loads with the same
4321  // element type or scalar loads and then recombines it to the widen vector
4322  // type.
4323  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4324  unsigned WidenWidth = WidenVT.getSizeInBits();
4325  EVT LdVT = LD->getMemoryVT();
4326  SDLoc dl(LD);
4327  assert(LdVT.isVector() && WidenVT.isVector());
4328  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4329 
4330  // Load information
4331  SDValue Chain = LD->getChain();
4332  SDValue BasePtr = LD->getBasePtr();
4333  unsigned Align = LD->getAlignment();
4334  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4335  AAMDNodes AAInfo = LD->getAAInfo();
4336 
4337  int LdWidth = LdVT.getSizeInBits();
4338  int WidthDiff = WidenWidth - LdWidth;
4339  unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
4340 
4341  // Find the vector type that can load from.
4342  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4343  int NewVTWidth = NewVT.getSizeInBits();
4344  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4345  Align, MMOFlags, AAInfo);
4346  LdChain.push_back(LdOp.getValue(1));
4347 
4348  // Check if we can load the element with one instruction.
4349  if (LdWidth <= NewVTWidth) {
4350  if (!NewVT.isVector()) {
4351  unsigned NumElts = WidenWidth / NewVTWidth;
4352  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4353  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4354  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4355  }
4356  if (NewVT == WidenVT)
4357  return LdOp;
4358 
4359  assert(WidenWidth % NewVTWidth == 0);
4360  unsigned NumConcat = WidenWidth / NewVTWidth;
4361  SmallVector<SDValue, 16> ConcatOps(NumConcat);
4362  SDValue UndefVal = DAG.getUNDEF(NewVT);
4363  ConcatOps[0] = LdOp;
4364  for (unsigned i = 1; i != NumConcat; ++i)
4365  ConcatOps[i] = UndefVal;
4366  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4367  }
4368 
4369  // Load vector by using multiple loads from largest vector to scalar.
4371  LdOps.push_back(LdOp);
4372 
4373  LdWidth -= NewVTWidth;
4374  unsigned Offset = 0;
4375 
4376  while (LdWidth > 0) {
4377  unsigned Increment = NewVTWidth / 8;
4378  Offset += Increment;
4379  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4380 
4381  SDValue L;
4382  if (LdWidth < NewVTWidth) {
4383  // The current type we are using is too large. Find a better size.
4384  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4385  NewVTWidth = NewVT.getSizeInBits();
4386  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4387  LD->getPointerInfo().getWithOffset(Offset),
4388  MinAlign(Align, Increment), MMOFlags, AAInfo);
4389  LdChain.push_back(L.getValue(1));
4390  if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
4391  // Later code assumes the vector loads produced will be mergeable, so we
4392  // must pad the final entry up to the previous width. Scalars are
4393  // combined separately.
4395  Loads.push_back(L);
4396  unsigned size = L->getValueSizeInBits(0);
4397  while (size < LdOp->getValueSizeInBits(0)) {
4398  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4399  size += L->getValueSizeInBits(0);
4400  }
4401  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4402  }
4403  } else {
4404  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4405  LD->getPointerInfo().getWithOffset(Offset),
4406  MinAlign(Align, Increment), MMOFlags, AAInfo);
4407  LdChain.push_back(L.getValue(1));
4408  }
4409 
4410  LdOps.push_back(L);
4411  LdOp = L;
4412 
4413  LdWidth -= NewVTWidth;
4414  }
4415 
4416  // Build the vector from the load operations.
4417  unsigned End = LdOps.size();
4418  if (!LdOps[0].getValueType().isVector())
4419  // All the loads are scalar loads.
4420  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4421 
4422  // If the load contains vectors, build the vector using concat vector.
4423  // All of the vectors used to load are power-of-2, and the scalar loads can be
4424  // combined to make a power-of-2 vector.
4425  SmallVector<SDValue, 16> ConcatOps(End);
4426  int i = End - 1;
4427  int Idx = End;
4428  EVT LdTy = LdOps[i].getValueType();
4429  // First, combine the scalar loads to a vector.
4430  if (!LdTy.isVector()) {
4431  for (--i; i >= 0; --i) {
4432  LdTy = LdOps[i].getValueType();
4433  if (LdTy.isVector())
4434  break;
4435  }
4436  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4437  }
4438  ConcatOps[--Idx] = LdOps[i];
4439  for (--i; i >= 0; --i) {
4440  EVT NewLdTy = LdOps[i].getValueType();
4441  if (NewLdTy != LdTy) {
4442  // Create a larger vector.
4443  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4444  makeArrayRef(&ConcatOps[Idx], End - Idx));
4445  Idx = End - 1;
4446  LdTy = NewLdTy;
4447  }
4448  ConcatOps[--Idx] = LdOps[i];
4449  }
4450 
4451  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
4452  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4453  makeArrayRef(&ConcatOps[Idx], End - Idx));
4454 
4455  // We need to fill the rest with undefs to build the vector.
4456  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
4457  SmallVector<SDValue, 16> WidenOps(NumOps);
4458  SDValue UndefVal = DAG.getUNDEF(LdTy);
4459  {
4460  unsigned i = 0;
4461  for (; i != End-Idx; ++i)
4462  WidenOps[i] = ConcatOps[Idx+i];
4463  for (; i != NumOps; ++i)
4464  WidenOps[i] = UndefVal;
4465  }
4466  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
4467 }
4468 
4469 SDValue
4470 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
4471  LoadSDNode *LD,
4473  // For extension loads, it may not be more efficient to chop up the vector
4474  // and then extend it. Instead, we unroll the load and build a new vector.
4475  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4476  EVT LdVT = LD->getMemoryVT();
4477  SDLoc dl(LD);
4478  assert(LdVT.isVector() && WidenVT.isVector());
4479 
4480  // Load information
4481  SDValue Chain = LD->getChain();
4482  SDValue BasePtr = LD->getBasePtr();
4483  unsigned Align = LD->getAlignment();
4484  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4485  AAMDNodes AAInfo = LD->getAAInfo();
4486 
4487  EVT EltVT = WidenVT.getVectorElementType();
4488  EVT LdEltVT = LdVT.getVectorElementType();
4489  unsigned NumElts = LdVT.getVectorNumElements();
4490 
4491  // Load each element and widen.
4492  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4493  SmallVector<SDValue, 16> Ops(WidenNumElts);
4494  unsigned Increment = LdEltVT.getSizeInBits() / 8;
4495  Ops[0] =
4496  DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
4497  LdEltVT, Align, MMOFlags, AAInfo);
4498  LdChain.push_back(Ops[0].getValue(1));
4499  unsigned i = 0, Offset = Increment;
4500  for (i=1; i < NumElts; ++i, Offset += Increment) {
4501  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4502  Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
4503  LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
4504  Align, MMOFlags, AAInfo);
4505  LdChain.push_back(Ops[i].getValue(1));
4506  }
4507 
4508  // Fill the rest with undefs.
4509  SDValue UndefVal = DAG.getUNDEF(EltVT);
4510  for (; i != WidenNumElts; ++i)
4511  Ops[i] = UndefVal;
4512 
4513  return DAG.getBuildVector(WidenVT, dl, Ops);
4514 }
4515 
4516 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
4517  StoreSDNode *ST) {
4518  // The strategy assumes that we can efficiently store power-of-two widths.
4519  // The routine chops the vector into the largest vector stores with the same
4520  // element type or scalar stores.
4521  SDValue Chain = ST->getChain();
4522  SDValue BasePtr = ST->getBasePtr();
4523  unsigned Align = ST->getAlignment();
4524  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4525  AAMDNodes AAInfo = ST->getAAInfo();
4526  SDValue ValOp = GetWidenedVector(ST->getValue());
4527  SDLoc dl(ST);
4528 
4529  EVT StVT = ST->getMemoryVT();
4530  unsigned StWidth = StVT.getSizeInBits();
4531  EVT ValVT = ValOp.getValueType();
4532  unsigned ValWidth = ValVT.getSizeInBits();
4533  EVT ValEltVT = ValVT.getVectorElementType();
4534  unsigned ValEltWidth = ValEltVT.getSizeInBits();
4535  assert(StVT.getVectorElementType() == ValEltVT);
4536 
4537  int Idx = 0; // current index to store
4538  unsigned Offset = 0; // offset from base to store
4539  while (StWidth != 0) {
4540  // Find the largest vector type we can store with.
4541  EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
4542  unsigned NewVTWidth = NewVT.getSizeInBits();
4543  unsigned Increment = NewVTWidth / 8;
4544  if (NewVT.isVector()) {
4545  unsigned NumVTElts = NewVT.getVectorNumElements();
4546  do {
4547  SDValue EOp = DAG.getNode(
4548  ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
4549  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4550  StChain.push_back(DAG.getStore(
4551  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4552  MinAlign(Align, Offset), MMOFlags, AAInfo));
4553  StWidth -= NewVTWidth;
4554  Offset += Increment;
4555  Idx += NumVTElts;
4556 
4557  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4558  } while (StWidth != 0 && StWidth >= NewVTWidth);
4559  } else {
4560  // Cast the vector to the scalar type we can store.
4561  unsigned NumElts = ValWidth / NewVTWidth;
4562  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4563  SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4564  // Readjust index position based on new vector type.
4565  Idx = Idx * ValEltWidth / NewVTWidth;
4566  do {
4567  SDValue EOp = DAG.getNode(
4568  ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4569  DAG.getConstant(Idx++, dl,
4570  TLI.getVectorIdxTy(DAG.getDataLayout())));
4571  StChain.push_back(DAG.getStore(
4572  Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4573  MinAlign(Align, Offset), MMOFlags, AAInfo));
4574  StWidth -= NewVTWidth;
4575  Offset += Increment;
4576  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4577  } while (StWidth != 0 && StWidth >= NewVTWidth);
4578  // Restore index back to be relative to the original widen element type.
4579  Idx = Idx * NewVTWidth / ValEltWidth;
4580  }
4581  }
4582 }
4583 
4584 void
4585 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4586  StoreSDNode *ST) {
4587  // For extension loads, it may not be more efficient to truncate the vector
4588  // and then store it. Instead, we extract each element and then store it.
4589  SDValue Chain = ST->getChain();
4590  SDValue BasePtr = ST->getBasePtr();
4591  unsigned Align = ST->getAlignment();
4592  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4593  AAMDNodes AAInfo = ST->getAAInfo();
4594  SDValue ValOp = GetWidenedVector(ST->getValue());
4595  SDLoc dl(ST);
4596 
4597  EVT StVT = ST->getMemoryVT();
4598  EVT ValVT = ValOp.getValueType();
4599 
4600  // It must be true that the wide vector type is bigger than where we need to
4601  // store.
4602  assert(StVT.isVector() && ValOp.getValueType().isVector());
4603  assert(StVT.bitsLT(ValOp.getValueType()));
4604 
4605  // For truncating stores, we can not play the tricks of chopping legal vector
4606  // types and bitcast it to the right type. Instead, we unroll the store.
4607  EVT StEltVT = StVT.getVectorElementType();
4608  EVT ValEltVT = ValVT.getVectorElementType();
4609  unsigned Increment = ValEltVT.getSizeInBits() / 8;
4610  unsigned NumElts = StVT.getVectorNumElements();
4611  SDValue EOp = DAG.getNode(
4612  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4613  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4614  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4615  ST->getPointerInfo(), StEltVT, Align,
4616  MMOFlags, AAInfo));
4617  unsigned Offset = Increment;
4618  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4619  SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4620  SDValue EOp = DAG.getNode(
4621  ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4622  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4623  StChain.push_back(DAG.getTruncStore(
4624  Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4625  StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4626  }
4627 }
4628 
4629 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
4630 /// input vector must have the same element type as NVT.
4631 /// FillWithZeroes specifies that the vector should be widened with zeroes.
4632 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
4633  bool FillWithZeroes) {
4634  // Note that InOp might have been widened so it might already have
4635  // the right width or it might need be narrowed.
4636  EVT InVT = InOp.getValueType();
4638  "input and widen element type must match");
4639  SDLoc dl(InOp);
4640 
4641  // Check if InOp already has the right width.
4642  if (InVT == NVT)
4643  return InOp;
4644 
4645  unsigned InNumElts = InVT.getVectorNumElements();
4646  unsigned WidenNumElts = NVT.getVectorNumElements();
4647  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
4648  unsigned NumConcat = WidenNumElts / InNumElts;
4649  SmallVector<SDValue, 16> Ops(NumConcat);
4650  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
4651  DAG.getUNDEF(InVT);
4652  Ops[0] = InOp;
4653  for (unsigned i = 1; i != NumConcat; ++i)
4654  Ops[i] = FillVal;
4655 
4656  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
4657  }
4658 
4659  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
4660  return DAG.getNode(
4661  ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
4662  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4663 
4664  // Fall back to extract and build.
4665  SmallVector<SDValue, 16> Ops(WidenNumElts);
4666  EVT EltVT = NVT.getVectorElementType();
4667  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
4668  unsigned Idx;
4669  for (Idx = 0; Idx < MinNumElts; ++Idx)
4670  Ops[Idx] = DAG.getNode(
4671  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4672  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4673 
4674  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
4675  DAG.getUNDEF(EltVT);
4676  for ( ; Idx < WidenNumElts; ++Idx)
4677  Ops[Idx] = FillVal;
4678  return DAG.getBuildVector(NVT, dl, Ops);
4679 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:570
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:537
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:595
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isUndef() const
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:295
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:357
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, SmallVectorImpl< SDValue > &ConcatOps, unsigned ConcatEnd, EVT VT, EVT MaxVT, EVT WidenVT)
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0...
Definition: ISDOpcodes.h:606
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:366
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:518
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:350
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getChain() const
unsigned getAlignment() const
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:288
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:140
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:949
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:507
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:383
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:253
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:211
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:352
uint64_t High
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:434
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:135
A description of a memory reference used in the backend.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
Shift and rotation operations.
Definition: ISDOpcodes.h:409
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:201
This is used by foldAnyOrAllBitsSet() to capture a source va