LLVM  9.0.0svn
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
23 #include "llvm/IR/DataLayout.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Result Vector Scalarization: <1 x ty> -> ty.
32 //===----------------------------------------------------------------------===//
33 
34 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
35  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
36  dbgs() << "\n");
37  SDValue R = SDValue();
38 
39  switch (N->getOpcode()) {
40  default:
41 #ifndef NDEBUG
42  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
43  N->dump(&DAG);
44  dbgs() << "\n";
45 #endif
46  report_fatal_error("Do not know how to scalarize the result of this "
47  "operator!\n");
48 
49  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
50  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
51  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
52  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
53  case ISD::STRICT_FP_ROUND: R = ScalarizeVecRes_STRICT_FP_ROUND(N); break;
54  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
55  case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
56  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
70  R = ScalarizeVecRes_VecInregOp(N);
71  break;
72  case ISD::ABS:
73  case ISD::ANY_EXTEND:
74  case ISD::BITREVERSE:
75  case ISD::BSWAP:
76  case ISD::CTLZ:
78  case ISD::CTPOP:
79  case ISD::CTTZ:
81  case ISD::FABS:
82  case ISD::FCEIL:
83  case ISD::FCOS:
84  case ISD::FEXP:
85  case ISD::FEXP2:
86  case ISD::FFLOOR:
87  case ISD::FLOG:
88  case ISD::FLOG10:
89  case ISD::FLOG2:
90  case ISD::FNEARBYINT:
91  case ISD::FNEG:
92  case ISD::FP_EXTEND:
93  case ISD::FP_TO_SINT:
94  case ISD::FP_TO_UINT:
95  case ISD::FRINT:
96  case ISD::FROUND:
97  case ISD::FSIN:
98  case ISD::FSQRT:
99  case ISD::FTRUNC:
100  case ISD::SIGN_EXTEND:
101  case ISD::SINT_TO_FP:
102  case ISD::TRUNCATE:
103  case ISD::UINT_TO_FP:
104  case ISD::ZERO_EXTEND:
105  case ISD::FCANONICALIZE:
106  R = ScalarizeVecRes_UnaryOp(N);
107  break;
108 
109  case ISD::ADD:
110  case ISD::AND:
111  case ISD::FADD:
112  case ISD::FCOPYSIGN:
113  case ISD::FDIV:
114  case ISD::FMUL:
115  case ISD::FMINNUM:
116  case ISD::FMAXNUM:
117  case ISD::FMINNUM_IEEE:
118  case ISD::FMAXNUM_IEEE:
119  case ISD::FMINIMUM:
120  case ISD::FMAXIMUM:
121  case ISD::SMIN:
122  case ISD::SMAX:
123  case ISD::UMIN:
124  case ISD::UMAX:
125 
126  case ISD::SADDSAT:
127  case ISD::UADDSAT:
128  case ISD::SSUBSAT:
129  case ISD::USUBSAT:
130 
131  case ISD::FPOW:
132  case ISD::FREM:
133  case ISD::FSUB:
134  case ISD::MUL:
135  case ISD::OR:
136  case ISD::SDIV:
137  case ISD::SREM:
138  case ISD::SUB:
139  case ISD::UDIV:
140  case ISD::UREM:
141  case ISD::XOR:
142  case ISD::SHL:
143  case ISD::SRA:
144  case ISD::SRL:
145  R = ScalarizeVecRes_BinOp(N);
146  break;
147  case ISD::FMA:
148  R = ScalarizeVecRes_TernaryOp(N);
149  break;
150  case ISD::STRICT_FADD:
151  case ISD::STRICT_FSUB:
152  case ISD::STRICT_FMUL:
153  case ISD::STRICT_FDIV:
154  case ISD::STRICT_FREM:
155  case ISD::STRICT_FSQRT:
156  case ISD::STRICT_FMA:
157  case ISD::STRICT_FPOW:
158  case ISD::STRICT_FPOWI:
159  case ISD::STRICT_FSIN:
160  case ISD::STRICT_FCOS:
161  case ISD::STRICT_FEXP:
162  case ISD::STRICT_FEXP2:
163  case ISD::STRICT_FLOG:
164  case ISD::STRICT_FLOG10:
165  case ISD::STRICT_FLOG2:
166  case ISD::STRICT_FRINT:
168  case ISD::STRICT_FMAXNUM:
169  case ISD::STRICT_FMINNUM:
170  case ISD::STRICT_FCEIL:
171  case ISD::STRICT_FFLOOR:
172  case ISD::STRICT_FROUND:
173  case ISD::STRICT_FTRUNC:
175  R = ScalarizeVecRes_StrictFPOp(N);
176  break;
177  case ISD::UADDO:
178  case ISD::SADDO:
179  case ISD::USUBO:
180  case ISD::SSUBO:
181  case ISD::UMULO:
182  case ISD::SMULO:
183  R = ScalarizeVecRes_OverflowOp(N, ResNo);
184  break;
185  case ISD::SMULFIX:
186  case ISD::SMULFIXSAT:
187  case ISD::UMULFIX:
188  R = ScalarizeVecRes_MULFIX(N);
189  break;
190  }
191 
192  // If R is null, the sub-method took care of registering the result.
193  if (R.getNode())
194  SetScalarizedVector(SDValue(N, ResNo), R);
195 }
196 
197 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
198  SDValue LHS = GetScalarizedVector(N->getOperand(0));
199  SDValue RHS = GetScalarizedVector(N->getOperand(1));
200  return DAG.getNode(N->getOpcode(), SDLoc(N),
201  LHS.getValueType(), LHS, RHS, N->getFlags());
202 }
203 
204 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
205  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
206  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
207  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
208  return DAG.getNode(N->getOpcode(), SDLoc(N),
209  Op0.getValueType(), Op0, Op1, Op2);
210 }
211 
212 SDValue DAGTypeLegalizer::ScalarizeVecRes_MULFIX(SDNode *N) {
213  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
214  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
215  SDValue Op2 = N->getOperand(2);
216  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
217  Op2);
218 }
219 
220 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
222  unsigned NumOpers = N->getNumOperands();
223  SDValue Chain = N->getOperand(0);
224  EVT ValueVTs[] = {VT, MVT::Other};
225  SDLoc dl(N);
226 
228 
229  // The Chain is the first operand.
230  Opers.push_back(Chain);
231 
232  // Now process the remaining operands.
233  for (unsigned i = 1; i < NumOpers; ++i) {
234  SDValue Oper = N->getOperand(i);
235 
236  if (Oper.getValueType().isVector())
237  Oper = GetScalarizedVector(Oper);
238 
239  Opers.push_back(Oper);
240  }
241 
242  SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
243 
244  // Legalize the chain result - switch anything that used the old chain to
245  // use the new one.
246  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
247  return Result;
248 }
249 
250 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
251  unsigned ResNo) {
252  SDLoc DL(N);
253  EVT ResVT = N->getValueType(0);
254  EVT OvVT = N->getValueType(1);
255 
256  SDValue ScalarLHS, ScalarRHS;
257  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
258  ScalarLHS = GetScalarizedVector(N->getOperand(0));
259  ScalarRHS = GetScalarizedVector(N->getOperand(1));
260  } else {
261  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
262  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
263  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
264  ScalarLHS = ElemsLHS[0];
265  ScalarRHS = ElemsRHS[0];
266  }
267 
268  SDVTList ScalarVTs = DAG.getVTList(
270  SDNode *ScalarNode = DAG.getNode(
271  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
272 
273  // Replace the other vector result not being explicitly scalarized here.
274  unsigned OtherNo = 1 - ResNo;
275  EVT OtherVT = N->getValueType(OtherNo);
276  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
277  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
278  } else {
279  SDValue OtherVal = DAG.getNode(
280  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
281  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
282  }
283 
284  return SDValue(ScalarNode, ResNo);
285 }
286 
287 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
288  unsigned ResNo) {
289  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
290  return GetScalarizedVector(Op);
291 }
292 
293 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
294  SDValue Op = N->getOperand(0);
295  if (Op.getValueType().isVector()
296  && Op.getValueType().getVectorNumElements() == 1
297  && !isSimpleLegalType(Op.getValueType()))
298  Op = GetScalarizedVector(Op);
299  EVT NewVT = N->getValueType(0).getVectorElementType();
300  return DAG.getNode(ISD::BITCAST, SDLoc(N),
301  NewVT, Op);
302 }
303 
304 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
305  EVT EltVT = N->getValueType(0).getVectorElementType();
306  SDValue InOp = N->getOperand(0);
307  // The BUILD_VECTOR operands may be of wider element types and
308  // we may need to truncate them back to the requested return type.
309  if (EltVT.isInteger())
310  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
311  return InOp;
312 }
313 
314 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
315  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
317  N->getOperand(0), N->getOperand(1));
318 }
319 
320 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
321  EVT NewVT = N->getValueType(0).getVectorElementType();
322  SDValue Op = GetScalarizedVector(N->getOperand(0));
323  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
324  NewVT, Op, N->getOperand(1));
325 }
326 
327 SDValue DAGTypeLegalizer::ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N) {
328  EVT NewVT = N->getValueType(0).getVectorElementType();
329  SDValue Op = GetScalarizedVector(N->getOperand(1));
331  { NewVT, MVT::Other },
332  { N->getOperand(0), Op, N->getOperand(2) });
333  // Legalize the chain result - switch anything that used the old chain to
334  // use the new one.
335  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
336  return Res;
337 }
338 
339 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
340  SDValue Op = GetScalarizedVector(N->getOperand(0));
341  return DAG.getNode(ISD::FPOWI, SDLoc(N),
342  Op.getValueType(), Op, N->getOperand(1));
343 }
344 
345 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
346  // The value to insert may have a wider type than the vector element type,
347  // so be sure to truncate it to the element type if necessary.
348  SDValue Op = N->getOperand(1);
349  EVT EltVT = N->getValueType(0).getVectorElementType();
350  if (Op.getValueType() != EltVT)
351  // FIXME: Can this happen for floating point types?
352  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
353  return Op;
354 }
355 
356 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
357  assert(N->isUnindexed() && "Indexed vector load?");
358 
359  SDValue Result = DAG.getLoad(
362  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
365  N->getAAInfo());
366 
367  // Legalize the chain result - switch anything that used the old chain to
368  // use the new one.
369  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
370  return Result;
371 }
372 
373 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
374  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
375  EVT DestVT = N->getValueType(0).getVectorElementType();
376  SDValue Op = N->getOperand(0);
377  EVT OpVT = Op.getValueType();
378  SDLoc DL(N);
379  // The result needs scalarizing, but it's not a given that the source does.
380  // This is a workaround for targets where it's impossible to scalarize the
381  // result of a conversion, because the source type is legal.
382  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
383  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
384  // legal and was not scalarized.
385  // See the similar logic in ScalarizeVecRes_SETCC
386  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
387  Op = GetScalarizedVector(Op);
388  } else {
389  EVT VT = OpVT.getVectorElementType();
390  Op = DAG.getNode(
391  ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
392  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
393  }
394  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
395 }
396 
397 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
398  EVT EltVT = N->getValueType(0).getVectorElementType();
399  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
400  SDValue LHS = GetScalarizedVector(N->getOperand(0));
401  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
402  LHS, DAG.getValueType(ExtVT));
403 }
404 
405 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
406  SDLoc DL(N);
407  SDValue Op = N->getOperand(0);
408 
409  EVT OpVT = Op.getValueType();
410  EVT OpEltVT = OpVT.getVectorElementType();
411  EVT EltVT = N->getValueType(0).getVectorElementType();
412 
413  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
414  Op = GetScalarizedVector(Op);
415  } else {
416  Op = DAG.getNode(
417  ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
418  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
419  }
420 
421  switch (N->getOpcode()) {
423  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
425  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
427  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
428  }
429 
430  llvm_unreachable("Illegal extend_vector_inreg opcode");
431 }
432 
433 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
434  // If the operand is wider than the vector element type then it is implicitly
435  // truncated. Make that explicit here.
436  EVT EltVT = N->getValueType(0).getVectorElementType();
437  SDValue InOp = N->getOperand(0);
438  if (InOp.getValueType() != EltVT)
439  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
440  return InOp;
441 }
442 
443 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
444  SDValue Cond = N->getOperand(0);
445  EVT OpVT = Cond.getValueType();
446  SDLoc DL(N);
447  // The vselect result and true/value operands needs scalarizing, but it's
448  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
449  // See the similar logic in ScalarizeVecRes_SETCC
450  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
451  Cond = GetScalarizedVector(Cond);
452  } else {
453  EVT VT = OpVT.getVectorElementType();
454  Cond = DAG.getNode(
455  ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
456  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
457  }
458 
459  SDValue LHS = GetScalarizedVector(N->getOperand(1));
460  TargetLowering::BooleanContent ScalarBool =
461  TLI.getBooleanContents(false, false);
462  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
463 
464  // If integer and float booleans have different contents then we can't
465  // reliably optimize in all cases. There is a full explanation for this in
466  // DAGCombiner::visitSELECT() where the same issue affects folding
467  // (select C, 0, 1) to (xor C, 1).
468  if (TLI.getBooleanContents(false, false) !=
469  TLI.getBooleanContents(false, true)) {
470  // At least try the common case where the boolean is generated by a
471  // comparison.
472  if (Cond->getOpcode() == ISD::SETCC) {
473  EVT OpVT = Cond->getOperand(0).getValueType();
474  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
475  VecBool = TLI.getBooleanContents(OpVT);
476  } else
478  }
479 
480  EVT CondVT = Cond.getValueType();
481  if (ScalarBool != VecBool) {
482  switch (ScalarBool) {
484  break;
488  // Vector read from all ones, scalar expects a single 1 so mask.
489  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
490  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
491  break;
495  // Vector reads from a one, scalar from all ones so sign extend.
496  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
497  Cond, DAG.getValueType(MVT::i1));
498  break;
499  }
500  }
501 
502  // Truncate the condition if needed
503  auto BoolVT = getSetCCResultType(CondVT);
504  if (BoolVT.bitsLT(CondVT))
505  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
506 
507  return DAG.getSelect(SDLoc(N),
508  LHS.getValueType(), Cond, LHS,
509  GetScalarizedVector(N->getOperand(2)));
510 }
511 
512 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
513  SDValue LHS = GetScalarizedVector(N->getOperand(1));
514  return DAG.getSelect(SDLoc(N),
515  LHS.getValueType(), N->getOperand(0), LHS,
516  GetScalarizedVector(N->getOperand(2)));
517 }
518 
519 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
520  SDValue LHS = GetScalarizedVector(N->getOperand(2));
521  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
522  N->getOperand(0), N->getOperand(1),
523  LHS, GetScalarizedVector(N->getOperand(3)),
524  N->getOperand(4));
525 }
526 
527 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
528  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
529 }
530 
531 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
532  // Figure out if the scalar is the LHS or RHS and return it.
533  SDValue Arg = N->getOperand(2).getOperand(0);
534  if (Arg.isUndef())
535  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
536  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
537  return GetScalarizedVector(N->getOperand(Op));
538 }
539 
540 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
541  assert(N->getValueType(0).isVector() &&
542  N->getOperand(0).getValueType().isVector() &&
543  "Operand types must be vectors");
544  SDValue LHS = N->getOperand(0);
545  SDValue RHS = N->getOperand(1);
546  EVT OpVT = LHS.getValueType();
547  EVT NVT = N->getValueType(0).getVectorElementType();
548  SDLoc DL(N);
549 
550  // The result needs scalarizing, but it's not a given that the source does.
551  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
552  LHS = GetScalarizedVector(LHS);
553  RHS = GetScalarizedVector(RHS);
554  } else {
555  EVT VT = OpVT.getVectorElementType();
556  LHS = DAG.getNode(
557  ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
558  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
559  RHS = DAG.getNode(
560  ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
561  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
562  }
563 
564  // Turn it into a scalar SETCC.
565  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
566  N->getOperand(2));
567  // Vectors may have a different boolean contents to scalars. Promote the
568  // value appropriately.
569  ISD::NodeType ExtendCode =
571  return DAG.getNode(ExtendCode, DL, NVT, Res);
572 }
573 
574 
575 //===----------------------------------------------------------------------===//
576 // Operand Vector Scalarization <1 x ty> -> ty.
577 //===----------------------------------------------------------------------===//
578 
579 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
580  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
581  dbgs() << "\n");
582  SDValue Res = SDValue();
583 
584  if (!Res.getNode()) {
585  switch (N->getOpcode()) {
586  default:
587 #ifndef NDEBUG
588  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
589  N->dump(&DAG);
590  dbgs() << "\n";
591 #endif
592  report_fatal_error("Do not know how to scalarize this operator's "
593  "operand!\n");
594  case ISD::BITCAST:
595  Res = ScalarizeVecOp_BITCAST(N);
596  break;
597  case ISD::ANY_EXTEND:
598  case ISD::ZERO_EXTEND:
599  case ISD::SIGN_EXTEND:
600  case ISD::TRUNCATE:
601  case ISD::FP_TO_SINT:
602  case ISD::FP_TO_UINT:
603  case ISD::SINT_TO_FP:
604  case ISD::UINT_TO_FP:
605  Res = ScalarizeVecOp_UnaryOp(N);
606  break;
607  case ISD::CONCAT_VECTORS:
608  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
609  break;
611  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
612  break;
613  case ISD::VSELECT:
614  Res = ScalarizeVecOp_VSELECT(N);
615  break;
616  case ISD::SETCC:
617  Res = ScalarizeVecOp_VSETCC(N);
618  break;
619  case ISD::STORE:
620  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
621  break;
623  Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
624  break;
625  case ISD::FP_ROUND:
626  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
627  break;
628  case ISD::VECREDUCE_FADD:
629  case ISD::VECREDUCE_FMUL:
630  case ISD::VECREDUCE_ADD:
631  case ISD::VECREDUCE_MUL:
632  case ISD::VECREDUCE_AND:
633  case ISD::VECREDUCE_OR:
634  case ISD::VECREDUCE_XOR:
635  case ISD::VECREDUCE_SMAX:
636  case ISD::VECREDUCE_SMIN:
637  case ISD::VECREDUCE_UMAX:
638  case ISD::VECREDUCE_UMIN:
639  case ISD::VECREDUCE_FMAX:
640  case ISD::VECREDUCE_FMIN:
641  Res = ScalarizeVecOp_VECREDUCE(N);
642  break;
643  }
644  }
645 
646  // If the result is null, the sub-method took care of registering results etc.
647  if (!Res.getNode()) return false;
648 
649  // If the result is N, the sub-method updated N in place. Tell the legalizer
650  // core about this.
651  if (Res.getNode() == N)
652  return true;
653 
654  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
655  "Invalid operand expansion");
656 
657  ReplaceValueWith(SDValue(N, 0), Res);
658  return false;
659 }
660 
661 /// If the value to convert is a vector that needs to be scalarized, it must be
662 /// <1 x ty>. Convert the element instead.
663 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
664  SDValue Elt = GetScalarizedVector(N->getOperand(0));
665  return DAG.getNode(ISD::BITCAST, SDLoc(N),
666  N->getValueType(0), Elt);
667 }
668 
669 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
670 /// Do the operation on the element instead.
671 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
673  "Unexpected vector type!");
674  SDValue Elt = GetScalarizedVector(N->getOperand(0));
675  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
676  N->getValueType(0).getScalarType(), Elt);
677  // Revectorize the result so the types line up with what the uses of this
678  // expression expect.
679  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
680 }
681 
682 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
683 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
685  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
686  Ops[i] = GetScalarizedVector(N->getOperand(i));
687  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
688 }
689 
690 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
691 /// so just return the element, ignoring the index.
692 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
693  EVT VT = N->getValueType(0);
694  SDValue Res = GetScalarizedVector(N->getOperand(0));
695  if (Res.getValueType() != VT)
696  Res = VT.isFloatingPoint()
697  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
698  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
699  return Res;
700 }
701 
702 /// If the input condition is a vector that needs to be scalarized, it must be
703 /// <1 x i1>, so just convert to a normal ISD::SELECT
704 /// (still with vector output type since that was acceptable if we got here).
705 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
706  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
707  EVT VT = N->getValueType(0);
708 
709  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
710  N->getOperand(2));
711 }
712 
713 /// If the operand is a vector that needs to be scalarized then the
714 /// result must be v1i1, so just convert to a scalar SETCC and wrap
715 /// with a scalar_to_vector since the res type is legal if we got here
716 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
717  assert(N->getValueType(0).isVector() &&
718  N->getOperand(0).getValueType().isVector() &&
719  "Operand types must be vectors");
720  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
721 
722  EVT VT = N->getValueType(0);
723  SDValue LHS = GetScalarizedVector(N->getOperand(0));
724  SDValue RHS = GetScalarizedVector(N->getOperand(1));
725 
726  EVT OpVT = N->getOperand(0).getValueType();
727  EVT NVT = VT.getVectorElementType();
728  SDLoc DL(N);
729  // Turn it into a scalar SETCC.
730  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
731  N->getOperand(2));
732 
733  // Vectors may have a different boolean contents to scalars. Promote the
734  // value appropriately.
735  ISD::NodeType ExtendCode =
737 
738  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
739 
740  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
741 }
742 
743 /// If the value to store is a vector that needs to be scalarized, it must be
744 /// <1 x ty>. Just store the element.
745 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
746  assert(N->isUnindexed() && "Indexed store of one-element vector?");
747  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
748  SDLoc dl(N);
749 
750  if (N->isTruncatingStore())
751  return DAG.getTruncStore(
752  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
753  N->getBasePtr(), N->getPointerInfo(),
755  N->getMemOperand()->getFlags(), N->getAAInfo());
756 
757  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
758  N->getBasePtr(), N->getPointerInfo(),
760  N->getAAInfo());
761 }
762 
763 /// If the value to round is a vector that needs to be scalarized, it must be
764 /// <1 x ty>. Convert the element instead.
765 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
766  SDValue Elt = GetScalarizedVector(N->getOperand(0));
767  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
768  N->getValueType(0).getVectorElementType(), Elt,
769  N->getOperand(1));
770  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
771 }
772 
773 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
774  unsigned OpNo) {
775  assert(OpNo == 1 && "Wrong operand for scalarization!");
776  SDValue Elt = GetScalarizedVector(N->getOperand(1));
779  MVT::Other },
780  { N->getOperand(0), Elt, N->getOperand(2) });
781  // Legalize the chain result - switch anything that used the old chain to
782  // use the new one.
783  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
784  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
785 }
786 
787 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
788  SDValue Res = GetScalarizedVector(N->getOperand(0));
789  // Result type may be wider than element type.
790  if (Res.getValueType() != N->getValueType(0))
791  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
792  return Res;
793 }
794 
795 //===----------------------------------------------------------------------===//
796 // Result Vector Splitting
797 //===----------------------------------------------------------------------===//
798 
799 /// This method is called when the specified result of the specified node is
800 /// found to need vector splitting. At this point, the node may also have
801 /// invalid operands or may have other results that need legalization, we just
802 /// know that (at least) one result needs vector splitting.
803 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
804  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
805  SDValue Lo, Hi;
806 
807  // See if the target wants to custom expand this node.
808  if (CustomLowerNode(N, N->getValueType(ResNo), true))
809  return;
810 
811  switch (N->getOpcode()) {
812  default:
813 #ifndef NDEBUG
814  dbgs() << "SplitVectorResult #" << ResNo << ": ";
815  N->dump(&DAG);
816  dbgs() << "\n";
817 #endif
818  report_fatal_error("Do not know how to split the result of this "
819  "operator!\n");
820 
821  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
822  case ISD::VSELECT:
823  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
824  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
825  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
826  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
827  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
828  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
829  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
830  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
831  case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
832  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
833  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
834  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
835  case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
836  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
837  case ISD::LOAD:
838  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
839  break;
840  case ISD::MLOAD:
841  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
842  break;
843  case ISD::MGATHER:
844  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
845  break;
846  case ISD::SETCC:
847  SplitVecRes_SETCC(N, Lo, Hi);
848  break;
849  case ISD::VECTOR_SHUFFLE:
850  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
851  break;
852 
856  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
857  break;
858 
859  case ISD::ABS:
860  case ISD::BITREVERSE:
861  case ISD::BSWAP:
862  case ISD::CTLZ:
863  case ISD::CTTZ:
866  case ISD::CTPOP:
867  case ISD::FABS:
868  case ISD::FCEIL:
869  case ISD::FCOS:
870  case ISD::FEXP:
871  case ISD::FEXP2:
872  case ISD::FFLOOR:
873  case ISD::FLOG:
874  case ISD::FLOG10:
875  case ISD::FLOG2:
876  case ISD::FNEARBYINT:
877  case ISD::FNEG:
878  case ISD::FP_EXTEND:
880  case ISD::FP_ROUND:
882  case ISD::FP_TO_SINT:
883  case ISD::FP_TO_UINT:
884  case ISD::FRINT:
885  case ISD::FROUND:
886  case ISD::FSIN:
887  case ISD::FSQRT:
888  case ISD::FTRUNC:
889  case ISD::SINT_TO_FP:
890  case ISD::TRUNCATE:
891  case ISD::UINT_TO_FP:
892  case ISD::FCANONICALIZE:
893  SplitVecRes_UnaryOp(N, Lo, Hi);
894  break;
895 
896  case ISD::ANY_EXTEND:
897  case ISD::SIGN_EXTEND:
898  case ISD::ZERO_EXTEND:
899  SplitVecRes_ExtendOp(N, Lo, Hi);
900  break;
901 
902  case ISD::ADD:
903  case ISD::SUB:
904  case ISD::MUL:
905  case ISD::MULHS:
906  case ISD::MULHU:
907  case ISD::FADD:
908  case ISD::FSUB:
909  case ISD::FMUL:
910  case ISD::FMINNUM:
911  case ISD::FMAXNUM:
912  case ISD::FMINIMUM:
913  case ISD::FMAXIMUM:
914  case ISD::SDIV:
915  case ISD::UDIV:
916  case ISD::FDIV:
917  case ISD::FPOW:
918  case ISD::AND:
919  case ISD::OR:
920  case ISD::XOR:
921  case ISD::SHL:
922  case ISD::SRA:
923  case ISD::SRL:
924  case ISD::UREM:
925  case ISD::SREM:
926  case ISD::FREM:
927  case ISD::SMIN:
928  case ISD::SMAX:
929  case ISD::UMIN:
930  case ISD::UMAX:
931  case ISD::SADDSAT:
932  case ISD::UADDSAT:
933  case ISD::SSUBSAT:
934  case ISD::USUBSAT:
935  SplitVecRes_BinOp(N, Lo, Hi);
936  break;
937  case ISD::FMA:
938  SplitVecRes_TernaryOp(N, Lo, Hi);
939  break;
940  case ISD::STRICT_FADD:
941  case ISD::STRICT_FSUB:
942  case ISD::STRICT_FMUL:
943  case ISD::STRICT_FDIV:
944  case ISD::STRICT_FREM:
945  case ISD::STRICT_FSQRT:
946  case ISD::STRICT_FMA:
947  case ISD::STRICT_FPOW:
948  case ISD::STRICT_FPOWI:
949  case ISD::STRICT_FSIN:
950  case ISD::STRICT_FCOS:
951  case ISD::STRICT_FEXP:
952  case ISD::STRICT_FEXP2:
953  case ISD::STRICT_FLOG:
954  case ISD::STRICT_FLOG10:
955  case ISD::STRICT_FLOG2:
956  case ISD::STRICT_FRINT:
958  case ISD::STRICT_FMAXNUM:
959  case ISD::STRICT_FMINNUM:
960  case ISD::STRICT_FCEIL:
961  case ISD::STRICT_FFLOOR:
962  case ISD::STRICT_FROUND:
963  case ISD::STRICT_FTRUNC:
964  SplitVecRes_StrictFPOp(N, Lo, Hi);
965  break;
966  case ISD::UADDO:
967  case ISD::SADDO:
968  case ISD::USUBO:
969  case ISD::SSUBO:
970  case ISD::UMULO:
971  case ISD::SMULO:
972  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
973  break;
974  case ISD::SMULFIX:
975  case ISD::SMULFIXSAT:
976  case ISD::UMULFIX:
977  SplitVecRes_MULFIX(N, Lo, Hi);
978  break;
979  }
980 
981  // If Lo/Hi is null, the sub-method took care of registering results etc.
982  if (Lo.getNode())
983  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
984 }
985 
986 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
987  SDValue &Hi) {
988  SDValue LHSLo, LHSHi;
989  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
990  SDValue RHSLo, RHSHi;
991  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
992  SDLoc dl(N);
993 
994  const SDNodeFlags Flags = N->getFlags();
995  unsigned Opcode = N->getOpcode();
996  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
997  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
998 }
999 
1000 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1001  SDValue &Hi) {
1002  SDValue Op0Lo, Op0Hi;
1003  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1004  SDValue Op1Lo, Op1Hi;
1005  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1006  SDValue Op2Lo, Op2Hi;
1007  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1008  SDLoc dl(N);
1009 
1010  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
1011  Op0Lo, Op1Lo, Op2Lo);
1012  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
1013  Op0Hi, Op1Hi, Op2Hi);
1014 }
1015 
1016 void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1017  SDValue LHSLo, LHSHi;
1018  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1019  SDValue RHSLo, RHSHi;
1020  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1021  SDLoc dl(N);
1022  SDValue Op2 = N->getOperand(2);
1023 
1024  unsigned Opcode = N->getOpcode();
1025  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
1026  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
1027 }
1028 
1029 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1030  SDValue &Hi) {
1031  // We know the result is a vector. The input may be either a vector or a
1032  // scalar value.
1033  EVT LoVT, HiVT;
1034  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1035  SDLoc dl(N);
1036 
1037  SDValue InOp = N->getOperand(0);
1038  EVT InVT = InOp.getValueType();
1039 
1040  // Handle some special cases efficiently.
1041  switch (getTypeAction(InVT)) {
1048  break;
1051  // A scalar to vector conversion, where the scalar needs expansion.
1052  // If the vector is being split in two then we can just convert the
1053  // expanded pieces.
1054  if (LoVT == HiVT) {
1055  GetExpandedOp(InOp, Lo, Hi);
1056  if (DAG.getDataLayout().isBigEndian())
1057  std::swap(Lo, Hi);
1058  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1059  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1060  return;
1061  }
1062  break;
1064  // If the input is a vector that needs to be split, convert each split
1065  // piece of the input now.
1066  GetSplitVector(InOp, Lo, Hi);
1067  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1068  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1069  return;
1070  }
1071 
1072  // In the general case, convert the input to an integer and split it by hand.
1073  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1074  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1075  if (DAG.getDataLayout().isBigEndian())
1076  std::swap(LoIntVT, HiIntVT);
1077 
1078  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1079 
1080  if (DAG.getDataLayout().isBigEndian())
1081  std::swap(Lo, Hi);
1082  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1083  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1084 }
1085 
1086 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1087  SDValue &Hi) {
1088  EVT LoVT, HiVT;
1089  SDLoc dl(N);
1090  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1091  unsigned LoNumElts = LoVT.getVectorNumElements();
1092  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1093  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1094 
1095  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1096  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1097 }
1098 
1099 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1100  SDValue &Hi) {
1101  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1102  SDLoc dl(N);
1103  unsigned NumSubvectors = N->getNumOperands() / 2;
1104  if (NumSubvectors == 1) {
1105  Lo = N->getOperand(0);
1106  Hi = N->getOperand(1);
1107  return;
1108  }
1109 
1110  EVT LoVT, HiVT;
1111  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1112 
1113  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1114  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1115 
1116  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1117  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1118 }
1119 
1120 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1121  SDValue &Hi) {
1122  SDValue Vec = N->getOperand(0);
1123  SDValue Idx = N->getOperand(1);
1124  SDLoc dl(N);
1125 
1126  EVT LoVT, HiVT;
1127  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1128 
1129  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1130  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1131  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1132  DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
1133  TLI.getVectorIdxTy(DAG.getDataLayout())));
1134 }
1135 
1136 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1137  SDValue &Hi) {
1138  SDValue Vec = N->getOperand(0);
1139  SDValue SubVec = N->getOperand(1);
1140  SDValue Idx = N->getOperand(2);
1141  SDLoc dl(N);
1142  GetSplitVector(Vec, Lo, Hi);
1143 
1144  EVT VecVT = Vec.getValueType();
1145  unsigned VecElems = VecVT.getVectorNumElements();
1146  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
1147 
1148  // If we know the index is 0, and we know the subvector doesn't cross the
1149  // boundary between the halves, we can avoid spilling the vector, and insert
1150  // into the lower half of the split vector directly.
1151  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
1152  // the index is constant and there is no boundary crossing. But those cases
1153  // don't seem to get hit in practice.
1154  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
1155  unsigned IdxVal = ConstIdx->getZExtValue();
1156  if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
1157  EVT LoVT, HiVT;
1158  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1159  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1160  return;
1161  }
1162  }
1163 
1164  // Spill the vector to the stack.
1165  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1166  SDValue Store =
1167  DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1168 
1169  // Store the new subvector into the specified index.
1170  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1171  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1172  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1173  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1174 
1175  // Load the Lo part from the stack slot.
1176  Lo =
1177  DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1178 
1179  // Increment the pointer to the other part.
1180  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1181  StackPtr =
1182  DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1183  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
1184 
1185  // Load the Hi part from the stack slot.
1186  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1187  MinAlign(Alignment, IncrementSize));
1188 }
1189 
1190 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1191  SDValue &Hi) {
1192  SDLoc dl(N);
1193  GetSplitVector(N->getOperand(0), Lo, Hi);
1194  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1195  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1196 }
1197 
1198 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1199  SDValue &Hi) {
1200  SDValue LHSLo, LHSHi;
1201  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1202  SDLoc DL(N);
1203 
1204  SDValue RHSLo, RHSHi;
1205  SDValue RHS = N->getOperand(1);
1206  EVT RHSVT = RHS.getValueType();
1207  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1208  GetSplitVector(RHS, RHSLo, RHSHi);
1209  else
1210  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1211 
1212 
1213  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1214  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1215 }
1216 
1217 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1218  SDValue &Hi) {
1219  SDValue LHSLo, LHSHi;
1220  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1221  SDLoc dl(N);
1222 
1223  EVT LoVT, HiVT;
1224  std::tie(LoVT, HiVT) =
1225  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1226 
1227  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1228  DAG.getValueType(LoVT));
1229  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1230  DAG.getValueType(HiVT));
1231 }
1232 
1233 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1234  SDValue &Hi) {
1235  unsigned Opcode = N->getOpcode();
1236  SDValue N0 = N->getOperand(0);
1237 
1238  SDLoc dl(N);
1239  SDValue InLo, InHi;
1240 
1241  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1242  GetSplitVector(N0, InLo, InHi);
1243  else
1244  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1245 
1246  EVT InLoVT = InLo.getValueType();
1247  unsigned InNumElements = InLoVT.getVectorNumElements();
1248 
1249  EVT OutLoVT, OutHiVT;
1250  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1251  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1252  assert((2 * OutNumElements) <= InNumElements &&
1253  "Illegal extend vector in reg split");
1254 
1255  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1256  // input vector (i.e. we only use InLo):
1257  // OutLo will extend the first OutNumElements from InLo.
1258  // OutHi will extend the next OutNumElements from InLo.
1259 
1260  // Shuffle the elements from InLo for OutHi into the bottom elements to
1261  // create a 'fake' InHi.
1262  SmallVector<int, 8> SplitHi(InNumElements, -1);
1263  for (unsigned i = 0; i != OutNumElements; ++i)
1264  SplitHi[i] = i + OutNumElements;
1265  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1266 
1267  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1268  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1269 }
1270 
1271 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1272  SDValue &Hi) {
1273  unsigned NumOps = N->getNumOperands();
1274  SDValue Chain = N->getOperand(0);
1275  EVT LoVT, HiVT;
1276  SDLoc dl(N);
1277  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1278 
1281 
1282  // The Chain is the first operand.
1283  OpsLo.push_back(Chain);
1284  OpsHi.push_back(Chain);
1285 
1286  // Now process the remaining operands.
1287  for (unsigned i = 1; i < NumOps; ++i) {
1288  SDValue Op = N->getOperand(i);
1289  SDValue OpLo = Op;
1290  SDValue OpHi = Op;
1291 
1292  EVT InVT = Op.getValueType();
1293  if (InVT.isVector()) {
1294  // If the input also splits, handle it directly for a
1295  // compile time speedup. Otherwise split it by hand.
1296  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1297  GetSplitVector(Op, OpLo, OpHi);
1298  else
1299  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1300  }
1301 
1302  OpsLo.push_back(OpLo);
1303  OpsHi.push_back(OpHi);
1304  }
1305 
1306  EVT LoValueVTs[] = {LoVT, MVT::Other};
1307  EVT HiValueVTs[] = {HiVT, MVT::Other};
1308  Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1309  Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1310 
1311  // Build a factor node to remember that this Op is independent of the
1312  // other one.
1313  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1314  Lo.getValue(1), Hi.getValue(1));
1315 
1316  // Legalize the chain result - switch anything that used the old chain to
1317  // use the new one.
1318  ReplaceValueWith(SDValue(N, 1), Chain);
1319 }
1320 
1321 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1322  SDValue Chain = N->getOperand(0);
1323  EVT VT = N->getValueType(0);
1324  unsigned NE = VT.getVectorNumElements();
1325  EVT EltVT = VT.getVectorElementType();
1326  SDLoc dl(N);
1327 
1328  SmallVector<SDValue, 8> Scalars;
1329  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1330 
1331  // If ResNE is 0, fully unroll the vector op.
1332  if (ResNE == 0)
1333  ResNE = NE;
1334  else if (NE > ResNE)
1335  NE = ResNE;
1336 
1337  //The results of each unrolled operation, including the chain.
1338  EVT ChainVTs[] = {EltVT, MVT::Other};
1339  SmallVector<SDValue, 8> Chains;
1340 
1341  unsigned i;
1342  for (i = 0; i != NE; ++i) {
1343  Operands[0] = Chain;
1344  for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1345  SDValue Operand = N->getOperand(j);
1346  EVT OperandVT = Operand.getValueType();
1347  if (OperandVT.isVector()) {
1348  EVT OperandEltVT = OperandVT.getVectorElementType();
1349  Operands[j] =
1350  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
1351  DAG.getConstant(i, dl, TLI.getVectorIdxTy(
1352  DAG.getDataLayout())));
1353  } else {
1354  Operands[j] = Operand;
1355  }
1356  }
1357  SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1358  Scalar.getNode()->setFlags(N->getFlags());
1359 
1360  //Add in the scalar as well as its chain value to the
1361  //result vectors.
1362  Scalars.push_back(Scalar);
1363  Chains.push_back(Scalar.getValue(1));
1364  }
1365 
1366  for (; i < ResNE; ++i)
1367  Scalars.push_back(DAG.getUNDEF(EltVT));
1368 
1369  // Build a new factor node to connect the chain back together.
1370  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1371  ReplaceValueWith(SDValue(N, 1), Chain);
1372 
1373  // Create a new BUILD_VECTOR node
1374  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1375  return DAG.getBuildVector(VecVT, dl, Scalars);
1376 }
1377 
1378 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1379  SDValue &Lo, SDValue &Hi) {
1380  SDLoc dl(N);
1381  EVT ResVT = N->getValueType(0);
1382  EVT OvVT = N->getValueType(1);
1383  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1384  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1385  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1386 
1387  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1388  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1389  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1390  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1391  } else {
1392  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1393  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1394  }
1395 
1396  unsigned Opcode = N->getOpcode();
1397  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1398  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1399  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1400  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1401 
1402  Lo = SDValue(LoNode, ResNo);
1403  Hi = SDValue(HiNode, ResNo);
1404 
1405  // Replace the other vector result not being explicitly split here.
1406  unsigned OtherNo = 1 - ResNo;
1407  EVT OtherVT = N->getValueType(OtherNo);
1408  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1409  SetSplitVector(SDValue(N, OtherNo),
1410  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1411  } else {
1412  SDValue OtherVal = DAG.getNode(
1413  ISD::CONCAT_VECTORS, dl, OtherVT,
1414  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1415  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1416  }
1417 }
1418 
1419 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1420  SDValue &Hi) {
1421  SDValue Vec = N->getOperand(0);
1422  SDValue Elt = N->getOperand(1);
1423  SDValue Idx = N->getOperand(2);
1424  SDLoc dl(N);
1425  GetSplitVector(Vec, Lo, Hi);
1426 
1427  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1428  unsigned IdxVal = CIdx->getZExtValue();
1429  unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1430  if (IdxVal < LoNumElts)
1431  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1432  Lo.getValueType(), Lo, Elt, Idx);
1433  else
1434  Hi =
1435  DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1436  DAG.getConstant(IdxVal - LoNumElts, dl,
1437  TLI.getVectorIdxTy(DAG.getDataLayout())));
1438  return;
1439  }
1440 
1441  // See if the target wants to custom expand this node.
1442  if (CustomLowerNode(N, N->getValueType(0), true))
1443  return;
1444 
1445  // Make the vector elements byte-addressable if they aren't already.
1446  EVT VecVT = Vec.getValueType();
1447  EVT EltVT = VecVT.getVectorElementType();
1448  if (VecVT.getScalarSizeInBits() < 8) {
1449  EltVT = MVT::i8;
1450  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1451  VecVT.getVectorNumElements());
1452  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1453  // Extend the element type to match if needed.
1454  if (EltVT.bitsGT(Elt.getValueType()))
1455  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1456  }
1457 
1458  // Spill the vector to the stack.
1459  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1460  auto &MF = DAG.getMachineFunction();
1461  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1462  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1463  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1464 
1465  // Store the new element. This may be larger than the vector element type,
1466  // so use a truncating store.
1467  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1468  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1469  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1470  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1472 
1473  EVT LoVT, HiVT;
1474  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1475 
1476  // Load the Lo part from the stack slot.
1477  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1478 
1479  // Increment the pointer to the other part.
1480  unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1481  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1482  DAG.getConstant(IncrementSize, dl,
1483  StackPtr.getValueType()));
1484 
1485  // Load the Hi part from the stack slot.
1486  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1487  PtrInfo.getWithOffset(IncrementSize),
1488  MinAlign(Alignment, IncrementSize));
1489 
1490  // If we adjusted the original type, we need to truncate the results.
1491  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1492  if (LoVT != Lo.getValueType())
1493  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1494  if (HiVT != Hi.getValueType())
1495  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1496 }
1497 
1498 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1499  SDValue &Hi) {
1500  EVT LoVT, HiVT;
1501  SDLoc dl(N);
1502  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1503  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1504  Hi = DAG.getUNDEF(HiVT);
1505 }
1506 
1507 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1508  SDValue &Hi) {
1509  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1510  EVT LoVT, HiVT;
1511  SDLoc dl(LD);
1512  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1513 
1515  SDValue Ch = LD->getChain();
1516  SDValue Ptr = LD->getBasePtr();
1517  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1518  EVT MemoryVT = LD->getMemoryVT();
1519  unsigned Alignment = LD->getOriginalAlignment();
1520  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1521  AAMDNodes AAInfo = LD->getAAInfo();
1522 
1523  EVT LoMemVT, HiMemVT;
1524  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1525 
1526  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1527  LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1528 
1529  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1530  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1531  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1532  LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1533  Alignment, MMOFlags, AAInfo);
1534 
1535  // Build a factor node to remember that this load is independent of the
1536  // other one.
1537  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1538  Hi.getValue(1));
1539 
1540  // Legalize the chain result - switch anything that used the old chain to
1541  // use the new one.
1542  ReplaceValueWith(SDValue(LD, 1), Ch);
1543 }
1544 
1545 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1546  SDValue &Lo, SDValue &Hi) {
1547  EVT LoVT, HiVT;
1548  SDLoc dl(MLD);
1549  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1550 
1551  SDValue Ch = MLD->getChain();
1552  SDValue Ptr = MLD->getBasePtr();
1553  SDValue Mask = MLD->getMask();
1554  SDValue PassThru = MLD->getPassThru();
1555  unsigned Alignment = MLD->getOriginalAlignment();
1557 
1558  // if Alignment is equal to the vector size,
1559  // take the half of it for the second part
1560  unsigned SecondHalfAlignment =
1561  (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1562  Alignment/2 : Alignment;
1563 
1564  // Split Mask operand
1565  SDValue MaskLo, MaskHi;
1566  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1567  GetSplitVector(Mask, MaskLo, MaskHi);
1568  else
1569  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1570 
1571  EVT MemoryVT = MLD->getMemoryVT();
1572  EVT LoMemVT, HiMemVT;
1573  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1574 
1575  SDValue PassThruLo, PassThruHi;
1576  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1577  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1578  else
1579  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1580 
1584  Alignment, MLD->getAAInfo(), MLD->getRanges());
1585 
1586  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
1587  ExtType, MLD->isExpandingLoad());
1588 
1589  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1590  MLD->isExpandingLoad());
1591  unsigned HiOffset = LoMemVT.getStoreSize();
1592 
1595  HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
1596  MLD->getRanges());
1597 
1598  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
1599  ExtType, MLD->isExpandingLoad());
1600 
1601  // Build a factor node to remember that this load is independent of the
1602  // other one.
1603  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1604  Hi.getValue(1));
1605 
1606  // Legalize the chain result - switch anything that used the old chain to
1607  // use the new one.
1608  ReplaceValueWith(SDValue(MLD, 1), Ch);
1609 
1610 }
1611 
1612 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1613  SDValue &Lo, SDValue &Hi) {
1614  EVT LoVT, HiVT;
1615  SDLoc dl(MGT);
1616  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1617 
1618  SDValue Ch = MGT->getChain();
1619  SDValue Ptr = MGT->getBasePtr();
1620  SDValue Mask = MGT->getMask();
1621  SDValue PassThru = MGT->getPassThru();
1622  SDValue Index = MGT->getIndex();
1623  SDValue Scale = MGT->getScale();
1624  unsigned Alignment = MGT->getOriginalAlignment();
1625 
1626  // Split Mask operand
1627  SDValue MaskLo, MaskHi;
1628  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1629  GetSplitVector(Mask, MaskLo, MaskHi);
1630  else
1631  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1632 
1633  EVT MemoryVT = MGT->getMemoryVT();
1634  EVT LoMemVT, HiMemVT;
1635  // Split MemoryVT
1636  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1637 
1638  SDValue PassThruLo, PassThruHi;
1639  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1640  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1641  else
1642  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1643 
1644  SDValue IndexHi, IndexLo;
1645  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1646  GetSplitVector(Index, IndexLo, IndexHi);
1647  else
1648  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1649 
1653  Alignment, MGT->getAAInfo(), MGT->getRanges());
1654 
1655  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1656  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1657  MMO);
1658 
1659  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1660  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1661  MMO);
1662 
1663  // Build a factor node to remember that this load is independent of the
1664  // other one.
1665  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1666  Hi.getValue(1));
1667 
1668  // Legalize the chain result - switch anything that used the old chain to
1669  // use the new one.
1670  ReplaceValueWith(SDValue(MGT, 1), Ch);
1671 }
1672 
1673 
1674 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1675  assert(N->getValueType(0).isVector() &&
1676  N->getOperand(0).getValueType().isVector() &&
1677  "Operand types must be vectors");
1678 
1679  EVT LoVT, HiVT;
1680  SDLoc DL(N);
1681  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1682 
1683  // If the input also splits, handle it directly. Otherwise split it by hand.
1684  SDValue LL, LH, RL, RH;
1685  if (getTypeAction(N->getOperand(0).getValueType()) ==
1687  GetSplitVector(N->getOperand(0), LL, LH);
1688  else
1689  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1690 
1691  if (getTypeAction(N->getOperand(1).getValueType()) ==
1693  GetSplitVector(N->getOperand(1), RL, RH);
1694  else
1695  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1696 
1697  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1698  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1699 }
1700 
1701 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1702  SDValue &Hi) {
1703  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1704  EVT LoVT, HiVT;
1705  SDLoc dl(N);
1706  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1707 
1708  // If the input also splits, handle it directly for a compile time speedup.
1709  // Otherwise split it by hand.
1710  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1711  EVT InVT = N->getOperand(OpNo).getValueType();
1712  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1713  GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1714  else
1715  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1716 
1717  if (N->getOpcode() == ISD::FP_ROUND) {
1718  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1719  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1720  } else if (N->getOpcode() == ISD::STRICT_FP_ROUND) {
1721  Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other },
1722  { N->getOperand(0), Lo, N->getOperand(2) });
1723  Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other },
1724  { N->getOperand(0), Hi, N->getOperand(2) });
1725  SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1726  Lo.getValue(1), Hi.getValue(1));
1727  ReplaceValueWith(SDValue(N, 1), NewChain);
1728  } else if (N->isStrictFPOpcode()) {
1729  Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other },
1730  { N->getOperand(0), Lo });
1731  Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other },
1732  { N->getOperand(0), Hi });
1733  // Legalize the chain result - switch anything that used the old chain to
1734  // use the new one.
1735  SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1736  Lo.getValue(1), Hi.getValue(1));
1737  ReplaceValueWith(SDValue(N, 1), NewChain);
1738  } else {
1739  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1740  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1741  }
1742 }
1743 
1744 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1745  SDValue &Hi) {
1746  SDLoc dl(N);
1747  EVT SrcVT = N->getOperand(0).getValueType();
1748  EVT DestVT = N->getValueType(0);
1749  EVT LoVT, HiVT;
1750  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1751 
1752  // We can do better than a generic split operation if the extend is doing
1753  // more than just doubling the width of the elements and the following are
1754  // true:
1755  // - The number of vector elements is even,
1756  // - the source type is legal,
1757  // - the type of a split source is illegal,
1758  // - the type of an extended (by doubling element size) source is legal, and
1759  // - the type of that extended source when split is legal.
1760  //
1761  // This won't necessarily completely legalize the operation, but it will
1762  // more effectively move in the right direction and prevent falling down
1763  // to scalarization in many cases due to the input vector being split too
1764  // far.
1765  unsigned NumElements = SrcVT.getVectorNumElements();
1766  if ((NumElements & 1) == 0 &&
1767  SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1768  LLVMContext &Ctx = *DAG.getContext();
1769  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1770  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1771 
1772  EVT SplitLoVT, SplitHiVT;
1773  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1774  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1775  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1776  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1777  N->dump(&DAG); dbgs() << "\n");
1778  // Extend the source vector by one step.
1779  SDValue NewSrc =
1780  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1781  // Get the low and high halves of the new, extended one step, vector.
1782  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1783  // Extend those vector halves the rest of the way.
1784  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1785  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1786  return;
1787  }
1788  }
1789  // Fall back to the generic unary operator splitting otherwise.
1790  SplitVecRes_UnaryOp(N, Lo, Hi);
1791 }
1792 
1793 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1794  SDValue &Lo, SDValue &Hi) {
1795  // The low and high parts of the original input give four input vectors.
1796  SDValue Inputs[4];
1797  SDLoc dl(N);
1798  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1799  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1800  EVT NewVT = Inputs[0].getValueType();
1801  unsigned NewElts = NewVT.getVectorNumElements();
1802 
1803  // If Lo or Hi uses elements from at most two of the four input vectors, then
1804  // express it as a vector shuffle of those two inputs. Otherwise extract the
1805  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1807  for (unsigned High = 0; High < 2; ++High) {
1808  SDValue &Output = High ? Hi : Lo;
1809 
1810  // Build a shuffle mask for the output, discovering on the fly which
1811  // input vectors to use as shuffle operands (recorded in InputUsed).
1812  // If building a suitable shuffle vector proves too hard, then bail
1813  // out with useBuildVector set.
1814  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1815  unsigned FirstMaskIdx = High * NewElts;
1816  bool useBuildVector = false;
1817  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1818  // The mask element. This indexes into the input.
1819  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1820 
1821  // The input vector this mask element indexes into.
1822  unsigned Input = (unsigned)Idx / NewElts;
1823 
1824  if (Input >= array_lengthof(Inputs)) {
1825  // The mask element does not index into any input vector.
1826  Ops.push_back(-1);
1827  continue;
1828  }
1829 
1830  // Turn the index into an offset from the start of the input vector.
1831  Idx -= Input * NewElts;
1832 
1833  // Find or create a shuffle vector operand to hold this input.
1834  unsigned OpNo;
1835  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1836  if (InputUsed[OpNo] == Input) {
1837  // This input vector is already an operand.
1838  break;
1839  } else if (InputUsed[OpNo] == -1U) {
1840  // Create a new operand for this input vector.
1841  InputUsed[OpNo] = Input;
1842  break;
1843  }
1844  }
1845 
1846  if (OpNo >= array_lengthof(InputUsed)) {
1847  // More than two input vectors used! Give up on trying to create a
1848  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1849  useBuildVector = true;
1850  break;
1851  }
1852 
1853  // Add the mask index for the new shuffle vector.
1854  Ops.push_back(Idx + OpNo * NewElts);
1855  }
1856 
1857  if (useBuildVector) {
1858  EVT EltVT = NewVT.getVectorElementType();
1860 
1861  // Extract the input elements by hand.
1862  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1863  // The mask element. This indexes into the input.
1864  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1865 
1866  // The input vector this mask element indexes into.
1867  unsigned Input = (unsigned)Idx / NewElts;
1868 
1869  if (Input >= array_lengthof(Inputs)) {
1870  // The mask element is "undef" or indexes off the end of the input.
1871  SVOps.push_back(DAG.getUNDEF(EltVT));
1872  continue;
1873  }
1874 
1875  // Turn the index into an offset from the start of the input vector.
1876  Idx -= Input * NewElts;
1877 
1878  // Extract the vector element by hand.
1879  SVOps.push_back(DAG.getNode(
1880  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1881  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1882  }
1883 
1884  // Construct the Lo/Hi output using a BUILD_VECTOR.
1885  Output = DAG.getBuildVector(NewVT, dl, SVOps);
1886  } else if (InputUsed[0] == -1U) {
1887  // No input vectors were used! The result is undefined.
1888  Output = DAG.getUNDEF(NewVT);
1889  } else {
1890  SDValue Op0 = Inputs[InputUsed[0]];
1891  // If only one input was used, use an undefined vector for the other.
1892  SDValue Op1 = InputUsed[1] == -1U ?
1893  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1894  // At least one input vector was used. Create a new shuffle vector.
1895  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1896  }
1897 
1898  Ops.clear();
1899  }
1900 }
1901 
1902 
1903 //===----------------------------------------------------------------------===//
1904 // Operand Vector Splitting
1905 //===----------------------------------------------------------------------===//
1906 
1907 /// This method is called when the specified operand of the specified node is
1908 /// found to need vector splitting. At this point, all of the result types of
1909 /// the node are known to be legal, but other operands of the node may need
1910 /// legalization as well as the specified one.
1911 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1912  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1913  SDValue Res = SDValue();
1914 
1915  // See if the target wants to custom split this node.
1916  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1917  return false;
1918 
1919  if (!Res.getNode()) {
1920  switch (N->getOpcode()) {
1921  default:
1922 #ifndef NDEBUG
1923  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1924  N->dump(&DAG);
1925  dbgs() << "\n";
1926 #endif
1927  report_fatal_error("Do not know how to split this operator's "
1928  "operand!\n");
1929 
1930  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1931  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1932  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1933  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1934  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1935  case ISD::TRUNCATE:
1936  Res = SplitVecOp_TruncateHelper(N);
1937  break;
1938  case ISD::STRICT_FP_ROUND:
1939  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1940  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1941  case ISD::STORE:
1942  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1943  break;
1944  case ISD::MSTORE:
1945  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1946  break;
1947  case ISD::MSCATTER:
1948  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1949  break;
1950  case ISD::MGATHER:
1951  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1952  break;
1953  case ISD::VSELECT:
1954  Res = SplitVecOp_VSELECT(N, OpNo);
1955  break;
1956  case ISD::SINT_TO_FP:
1957  case ISD::UINT_TO_FP:
1958  if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1959  Res = SplitVecOp_TruncateHelper(N);
1960  else
1961  Res = SplitVecOp_UnaryOp(N);
1962  break;
1963  case ISD::FP_TO_SINT:
1964  case ISD::FP_TO_UINT:
1965  case ISD::CTTZ:
1966  case ISD::CTLZ:
1967  case ISD::CTPOP:
1968  case ISD::STRICT_FP_EXTEND:
1969  case ISD::FP_EXTEND:
1970  case ISD::SIGN_EXTEND:
1971  case ISD::ZERO_EXTEND:
1972  case ISD::ANY_EXTEND:
1973  case ISD::FTRUNC:
1974  case ISD::FCANONICALIZE:
1975  Res = SplitVecOp_UnaryOp(N);
1976  break;
1977 
1981  Res = SplitVecOp_ExtVecInRegOp(N);
1982  break;
1983 
1984  case ISD::VECREDUCE_FADD:
1985  case ISD::VECREDUCE_FMUL:
1986  case ISD::VECREDUCE_ADD:
1987  case ISD::VECREDUCE_MUL:
1988  case ISD::VECREDUCE_AND:
1989  case ISD::VECREDUCE_OR:
1990  case ISD::VECREDUCE_XOR:
1991  case ISD::VECREDUCE_SMAX:
1992  case ISD::VECREDUCE_SMIN:
1993  case ISD::VECREDUCE_UMAX:
1994  case ISD::VECREDUCE_UMIN:
1995  case ISD::VECREDUCE_FMAX:
1996  case ISD::VECREDUCE_FMIN:
1997  Res = SplitVecOp_VECREDUCE(N, OpNo);
1998  break;
1999  }
2000  }
2001 
2002  // If the result is null, the sub-method took care of registering results etc.
2003  if (!Res.getNode()) return false;
2004 
2005  // If the result is N, the sub-method updated N in place. Tell the legalizer
2006  // core about this.
2007  if (Res.getNode() == N)
2008  return true;
2009 
2010  if (N->isStrictFPOpcode())
2011  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2012  "Invalid operand expansion");
2013  else
2014  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2015  "Invalid operand expansion");
2016 
2017  ReplaceValueWith(SDValue(N, 0), Res);
2018  return false;
2019 }
2020 
2021 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2022  // The only possibility for an illegal operand is the mask, since result type
2023  // legalization would have handled this node already otherwise.
2024  assert(OpNo == 0 && "Illegal operand must be mask");
2025 
2026  SDValue Mask = N->getOperand(0);
2027  SDValue Src0 = N->getOperand(1);
2028  SDValue Src1 = N->getOperand(2);
2029  EVT Src0VT = Src0.getValueType();
2030  SDLoc DL(N);
2031  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2032 
2033  SDValue Lo, Hi;
2034  GetSplitVector(N->getOperand(0), Lo, Hi);
2035  assert(Lo.getValueType() == Hi.getValueType() &&
2036  "Lo and Hi have differing types");
2037 
2038  EVT LoOpVT, HiOpVT;
2039  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2040  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2041 
2042  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2043  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2044  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2045  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2046 
2047  SDValue LoSelect =
2048  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2049  SDValue HiSelect =
2050  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2051 
2052  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2053 }
2054 
2055 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2056  EVT ResVT = N->getValueType(0);
2057  SDValue Lo, Hi;
2058  SDLoc dl(N);
2059 
2060  SDValue VecOp = N->getOperand(OpNo);
2061  EVT VecVT = VecOp.getValueType();
2062  assert(VecVT.isVector() && "Can only split reduce vector operand");
2063  GetSplitVector(VecOp, Lo, Hi);
2064  EVT LoOpVT, HiOpVT;
2065  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2066 
2067  bool NoNaN = N->getFlags().hasNoNaNs();
2068  unsigned CombineOpc = 0;
2069  switch (N->getOpcode()) {
2070  case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
2071  case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
2072  case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
2073  case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
2074  case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
2075  case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
2076  case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
2077  case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
2078  case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
2079  case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
2080  case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
2081  case ISD::VECREDUCE_FMAX:
2082  CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXIMUM;
2083  break;
2084  case ISD::VECREDUCE_FMIN:
2085  CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM;
2086  break;
2087  default:
2088  llvm_unreachable("Unexpected reduce ISD node");
2089  }
2090 
2091  // Use the appropriate scalar instruction on the split subvectors before
2092  // reducing the now partially reduced smaller vector.
2093  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2094  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2095 }
2096 
2097 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2098  // The result has a legal vector type, but the input needs splitting.
2099  EVT ResVT = N->getValueType(0);
2100  SDValue Lo, Hi;
2101  SDLoc dl(N);
2102  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2103  EVT InVT = Lo.getValueType();
2104 
2105  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2106  InVT.getVectorNumElements());
2107 
2108  if (N->isStrictFPOpcode()) {
2109  Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2110  { N->getOperand(0), Lo });
2111  Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2112  { N->getOperand(0), Hi });
2113 
2114  // Build a factor node to remember that this operation is independent
2115  // of the other one.
2116  SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2117  Hi.getValue(1));
2118 
2119  // Legalize the chain result - switch anything that used the old chain to
2120  // use the new one.
2121  ReplaceValueWith(SDValue(N, 1), Ch);
2122  } else {
2123  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2124  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2125  }
2126 
2127  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2128 }
2129 
2130 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2131  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2132  // end up being split all the way down to individual components. Convert the
2133  // split pieces into integers and reassemble.
2134  SDValue Lo, Hi;
2135  GetSplitVector(N->getOperand(0), Lo, Hi);
2136  Lo = BitConvertToInteger(Lo);
2137  Hi = BitConvertToInteger(Hi);
2138 
2139  if (DAG.getDataLayout().isBigEndian())
2140  std::swap(Lo, Hi);
2141 
2142  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2143  JoinIntegers(Lo, Hi));
2144 }
2145 
2146 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2147  // We know that the extracted result type is legal.
2148  EVT SubVT = N->getValueType(0);
2149  SDValue Idx = N->getOperand(1);
2150  SDLoc dl(N);
2151  SDValue Lo, Hi;
2152  GetSplitVector(N->getOperand(0), Lo, Hi);
2153 
2154  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2155  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2156 
2157  if (IdxVal < LoElts) {
2158  assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
2159  "Extracted subvector crosses vector split!");
2160  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2161  } else {
2162  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2163  DAG.getConstant(IdxVal - LoElts, dl,
2164  Idx.getValueType()));
2165  }
2166 }
2167 
2168 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2169  SDValue Vec = N->getOperand(0);
2170  SDValue Idx = N->getOperand(1);
2171  EVT VecVT = Vec.getValueType();
2172 
2173  if (isa<ConstantSDNode>(Idx)) {
2174  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2175 
2176  SDValue Lo, Hi;
2177  GetSplitVector(Vec, Lo, Hi);
2178 
2179  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2180 
2181  if (IdxVal < LoElts)
2182  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2183  return SDValue(DAG.UpdateNodeOperands(N, Hi,
2184  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2185  Idx.getValueType())), 0);
2186  }
2187 
2188  // See if the target wants to custom expand this node.
2189  if (CustomLowerNode(N, N->getValueType(0), true))
2190  return SDValue();
2191 
2192  // Make the vector elements byte-addressable if they aren't already.
2193  SDLoc dl(N);
2194  EVT EltVT = VecVT.getVectorElementType();
2195  if (VecVT.getScalarSizeInBits() < 8) {
2196  EltVT = MVT::i8;
2197  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2198  VecVT.getVectorNumElements());
2199  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2200  }
2201 
2202  // Store the vector to the stack.
2203  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
2204  auto &MF = DAG.getMachineFunction();
2205  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2206  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2207  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2208 
2209  // Load back the required element.
2210  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2211 
2212  // FIXME: This is to handle i1 vectors with elements promoted to i8.
2213  // i1 vector handling needs general improvement.
2214  if (N->getValueType(0).bitsLT(EltVT)) {
2215  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2217  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2218  }
2219 
2220  return DAG.getExtLoad(
2221  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2223 }
2224 
2225 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2226  SDValue Lo, Hi;
2227 
2228  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2229  // splitting the result has the same effect as splitting the input operand.
2230  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2231 
2232  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2233 }
2234 
2235 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2236  unsigned OpNo) {
2237  EVT LoVT, HiVT;
2238  SDLoc dl(MGT);
2239  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2240 
2241  SDValue Ch = MGT->getChain();
2242  SDValue Ptr = MGT->getBasePtr();
2243  SDValue Index = MGT->getIndex();
2244  SDValue Scale = MGT->getScale();
2245  SDValue Mask = MGT->getMask();
2246  SDValue PassThru = MGT->getPassThru();
2247  unsigned Alignment = MGT->getOriginalAlignment();
2248 
2249  SDValue MaskLo, MaskHi;
2250  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2251  // Split Mask operand
2252  GetSplitVector(Mask, MaskLo, MaskHi);
2253  else
2254  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2255 
2256  EVT MemoryVT = MGT->getMemoryVT();
2257  EVT LoMemVT, HiMemVT;
2258  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2259 
2260  SDValue PassThruLo, PassThruHi;
2261  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2262  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2263  else
2264  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2265 
2266  SDValue IndexHi, IndexLo;
2267  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2268  GetSplitVector(Index, IndexLo, IndexHi);
2269  else
2270  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2271 
2275  Alignment, MGT->getAAInfo(), MGT->getRanges());
2276 
2277  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2278  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2279  OpsLo, MMO);
2280 
2281  MMO = DAG.getMachineFunction().
2284  Alignment, MGT->getAAInfo(),
2285  MGT->getRanges());
2286 
2287  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2288  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2289  OpsHi, MMO);
2290 
2291  // Build a factor node to remember that this load is independent of the
2292  // other one.
2293  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2294  Hi.getValue(1));
2295 
2296  // Legalize the chain result - switch anything that used the old chain to
2297  // use the new one.
2298  ReplaceValueWith(SDValue(MGT, 1), Ch);
2299 
2300  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2301  Hi);
2302  ReplaceValueWith(SDValue(MGT, 0), Res);
2303  return SDValue();
2304 }
2305 
2306 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2307  unsigned OpNo) {
2308  SDValue Ch = N->getChain();
2309  SDValue Ptr = N->getBasePtr();
2310  SDValue Mask = N->getMask();
2311  SDValue Data = N->getValue();
2312  EVT MemoryVT = N->getMemoryVT();
2313  unsigned Alignment = N->getOriginalAlignment();
2314  SDLoc DL(N);
2315 
2316  EVT LoMemVT, HiMemVT;
2317  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2318 
2319  SDValue DataLo, DataHi;
2320  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2321  // Split Data operand
2322  GetSplitVector(Data, DataLo, DataHi);
2323  else
2324  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2325 
2326  SDValue MaskLo, MaskHi;
2327  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2328  // Split Mask operand
2329  GetSplitVector(Mask, MaskLo, MaskHi);
2330  else
2331  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2332 
2333  // if Alignment is equal to the vector size,
2334  // take the half of it for the second part
2335  unsigned SecondHalfAlignment =
2336  (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
2337  Alignment/2 : Alignment;
2338 
2339  SDValue Lo, Hi;
2343  Alignment, N->getAAInfo(), N->getRanges());
2344 
2345  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
2346  N->isTruncatingStore(),
2347  N->isCompressingStore());
2348 
2349  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2350  N->isCompressingStore());
2351  unsigned HiOffset = LoMemVT.getStoreSize();
2352 
2355  HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
2356  N->getRanges());
2357 
2358  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
2360 
2361  // Build a factor node to remember that this store is independent of the
2362  // other one.
2363  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2364 }
2365 
2366 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2367  unsigned OpNo) {
2368  SDValue Ch = N->getChain();
2369  SDValue Ptr = N->getBasePtr();
2370  SDValue Mask = N->getMask();
2371  SDValue Index = N->getIndex();
2372  SDValue Scale = N->getScale();
2373  SDValue Data = N->getValue();
2374  EVT MemoryVT = N->getMemoryVT();
2375  unsigned Alignment = N->getOriginalAlignment();
2376  SDLoc DL(N);
2377 
2378  // Split all operands
2379  EVT LoMemVT, HiMemVT;
2380  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2381 
2382  SDValue DataLo, DataHi;
2383  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2384  // Split Data operand
2385  GetSplitVector(Data, DataLo, DataHi);
2386  else
2387  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2388 
2389  SDValue MaskLo, MaskHi;
2390  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2391  // Split Mask operand
2392  GetSplitVector(Mask, MaskLo, MaskHi);
2393  else
2394  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2395 
2396  SDValue IndexHi, IndexLo;
2397  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2398  GetSplitVector(Index, IndexLo, IndexHi);
2399  else
2400  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2401 
2402  SDValue Lo;
2406  Alignment, N->getAAInfo(), N->getRanges());
2407 
2408  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2409  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2410  DL, OpsLo, MMO);
2411 
2412  MMO = DAG.getMachineFunction().
2415  Alignment, N->getAAInfo(), N->getRanges());
2416 
2417  // The order of the Scatter operation after split is well defined. The "Hi"
2418  // part comes after the "Lo". So these two operations should be chained one
2419  // after another.
2420  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2421  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2422  DL, OpsHi, MMO);
2423 }
2424 
2425 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2426  assert(N->isUnindexed() && "Indexed store of vector?");
2427  assert(OpNo == 1 && "Can only split the stored value");
2428  SDLoc DL(N);
2429 
2430  bool isTruncating = N->isTruncatingStore();
2431  SDValue Ch = N->getChain();
2432  SDValue Ptr = N->getBasePtr();
2433  EVT MemoryVT = N->getMemoryVT();
2434  unsigned Alignment = N->getOriginalAlignment();
2435  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2436  AAMDNodes AAInfo = N->getAAInfo();
2437  SDValue Lo, Hi;
2438  GetSplitVector(N->getOperand(1), Lo, Hi);
2439 
2440  EVT LoMemVT, HiMemVT;
2441  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2442 
2443  // Scalarize if the split halves are not byte-sized.
2444  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2445  return TLI.scalarizeVectorStore(N, DAG);
2446 
2447  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2448 
2449  if (isTruncating)
2450  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2451  Alignment, MMOFlags, AAInfo);
2452  else
2453  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2454  AAInfo);
2455 
2456  // Increment the pointer to the other half.
2457  Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2458 
2459  if (isTruncating)
2460  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2461  N->getPointerInfo().getWithOffset(IncrementSize),
2462  HiMemVT, Alignment, MMOFlags, AAInfo);
2463  else
2464  Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2465  N->getPointerInfo().getWithOffset(IncrementSize),
2466  Alignment, MMOFlags, AAInfo);
2467 
2468  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2469 }
2470 
2471 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2472  SDLoc DL(N);
2473 
2474  // The input operands all must have the same type, and we know the result
2475  // type is valid. Convert this to a buildvector which extracts all the
2476  // input elements.
2477  // TODO: If the input elements are power-two vectors, we could convert this to
2478  // a new CONCAT_VECTORS node with elements that are half-wide.
2480  EVT EltVT = N->getValueType(0).getVectorElementType();
2481  for (const SDValue &Op : N->op_values()) {
2482  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2483  i != e; ++i) {
2484  Elts.push_back(DAG.getNode(
2485  ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2486  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2487  }
2488  }
2489 
2490  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2491 }
2492 
2493 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2494  // The result type is legal, but the input type is illegal. If splitting
2495  // ends up with the result type of each half still being legal, just
2496  // do that. If, however, that would result in an illegal result type,
2497  // we can try to get more clever with power-two vectors. Specifically,
2498  // split the input type, but also widen the result element size, then
2499  // concatenate the halves and truncate again. For example, consider a target
2500  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2501  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2502  // %inlo = v4i32 extract_subvector %in, 0
2503  // %inhi = v4i32 extract_subvector %in, 4
2504  // %lo16 = v4i16 trunc v4i32 %inlo
2505  // %hi16 = v4i16 trunc v4i32 %inhi
2506  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2507  // %res = v8i8 trunc v8i16 %in16
2508  //
2509  // Without this transform, the original truncate would end up being
2510  // scalarized, which is pretty much always a last resort.
2511  SDValue InVec = N->getOperand(0);
2512  EVT InVT = InVec->getValueType(0);
2513  EVT OutVT = N->getValueType(0);
2514  unsigned NumElements = OutVT.getVectorNumElements();
2515  bool IsFloat = OutVT.isFloatingPoint();
2516 
2517  // Widening should have already made sure this is a power-two vector
2518  // if we're trying to split it at all. assert() that's true, just in case.
2519  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2520 
2521  unsigned InElementSize = InVT.getScalarSizeInBits();
2522  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2523 
2524  // Determine the split output VT. If its legal we can just split dirctly.
2525  EVT LoOutVT, HiOutVT;
2526  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2527  assert(LoOutVT == HiOutVT && "Unequal split?");
2528 
2529  // If the input elements are only 1/2 the width of the result elements,
2530  // just use the normal splitting. Our trick only work if there's room
2531  // to split more than once.
2532  if (isTypeLegal(LoOutVT) ||
2533  InElementSize <= OutElementSize * 2)
2534  return SplitVecOp_UnaryOp(N);
2535  SDLoc DL(N);
2536 
2537  // Don't touch if this will be scalarized.
2538  EVT FinalVT = InVT;
2539  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2540  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2541 
2542  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2543  return SplitVecOp_UnaryOp(N);
2544 
2545  // Get the split input vector.
2546  SDValue InLoVec, InHiVec;
2547  GetSplitVector(InVec, InLoVec, InHiVec);
2548 
2549  // Truncate them to 1/2 the element size.
2550  EVT HalfElementVT = IsFloat ?
2551  EVT::getFloatingPointVT(InElementSize/2) :
2552  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2553  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2554  NumElements/2);
2555  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2556  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2557  // Concatenate them to get the full intermediate truncation result.
2558  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2559  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2560  HalfHi);
2561  // Now finish up by truncating all the way down to the original result
2562  // type. This should normally be something that ends up being legal directly,
2563  // but in theory if a target has very wide vectors and an annoyingly
2564  // restricted set of legal types, this split can chain to build things up.
2565  return IsFloat
2566  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2567  DAG.getTargetConstant(
2568  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2569  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2570 }
2571 
2572 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2573  assert(N->getValueType(0).isVector() &&
2574  N->getOperand(0).getValueType().isVector() &&
2575  "Operand types must be vectors");
2576  // The result has a legal vector type, but the input needs splitting.
2577  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2578  SDLoc DL(N);
2579  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2580  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2581  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2582  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2583  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2584 
2585  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2586  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2587  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2588  return PromoteTargetBoolean(Con, N->getValueType(0));
2589 }
2590 
2591 
2592 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2593  // The result has a legal vector type, but the input needs splitting.
2594  EVT ResVT = N->getValueType(0);
2595  SDValue Lo, Hi;
2596  SDLoc DL(N);
2597  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2598  EVT InVT = Lo.getValueType();
2599 
2600  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2601  InVT.getVectorNumElements());
2602 
2603  if (N->isStrictFPOpcode()) {
2604  Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2605  { N->getOperand(0), Lo, N->getOperand(2) });
2606  Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2607  { N->getOperand(0), Hi, N->getOperand(2) });
2608  // Legalize the chain result - switch anything that used the old chain to
2609  // use the new one.
2610  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2611  Lo.getValue(1), Hi.getValue(1));
2612  ReplaceValueWith(SDValue(N, 1), NewChain);
2613  } else {
2614  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2615  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2616  }
2617 
2618  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2619 }
2620 
2621 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2622  // The result (and the first input) has a legal vector type, but the second
2623  // input needs splitting.
2624  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2625 }
2626 
2627 
2628 //===----------------------------------------------------------------------===//
2629 // Result Vector Widening
2630 //===----------------------------------------------------------------------===//
2631 
2632 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2633  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2634  dbgs() << "\n");
2635 
2636  // See if the target wants to custom widen this node.
2637  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2638  return;
2639 
2640  SDValue Res = SDValue();
2641  switch (N->getOpcode()) {
2642  default:
2643 #ifndef NDEBUG
2644  dbgs() << "WidenVectorResult #" << ResNo << ": ";
2645  N->dump(&DAG);
2646  dbgs() << "\n";
2647 #endif
2648  llvm_unreachable("Do not know how to widen the result of this operator!");
2649 
2650  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2651  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2652  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2653  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2654  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2655  case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2656  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2657  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2658  case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2659  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2660  case ISD::VSELECT:
2661  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2662  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2663  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2664  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2665  case ISD::VECTOR_SHUFFLE:
2666  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2667  break;
2668  case ISD::MLOAD:
2669  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2670  break;
2671  case ISD::MGATHER:
2672  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2673  break;
2674 
2675  case ISD::ADD:
2676  case ISD::AND:
2677  case ISD::MUL:
2678  case ISD::MULHS:
2679  case ISD::MULHU:
2680  case ISD::OR:
2681  case ISD::SUB:
2682  case ISD::XOR:
2683  case ISD::FMINNUM:
2684  case ISD::FMAXNUM:
2685  case ISD::FMINIMUM:
2686  case ISD::FMAXIMUM:
2687  case ISD::SMIN:
2688  case ISD::SMAX:
2689  case ISD::UMIN:
2690  case ISD::UMAX:
2691  case ISD::UADDSAT:
2692  case ISD::SADDSAT:
2693  case ISD::USUBSAT:
2694  case ISD::SSUBSAT:
2695  Res = WidenVecRes_Binary(N);
2696  break;
2697 
2698  case ISD::FADD:
2699  case ISD::FMUL:
2700  case ISD::FPOW:
2701  case ISD::FSUB:
2702  case ISD::FDIV:
2703  case ISD::FREM:
2704  case ISD::SDIV:
2705  case ISD::UDIV:
2706  case ISD::SREM:
2707  case ISD::UREM:
2708  Res = WidenVecRes_BinaryCanTrap(N);
2709  break;
2710 
2711  case ISD::STRICT_FADD:
2712  case ISD::STRICT_FSUB:
2713  case ISD::STRICT_FMUL:
2714  case ISD::STRICT_FDIV:
2715  case ISD::STRICT_FREM:
2716  case ISD::STRICT_FSQRT:
2717  case ISD::STRICT_FMA:
2718  case ISD::STRICT_FPOW:
2719  case ISD::STRICT_FPOWI:
2720  case ISD::STRICT_FSIN:
2721  case ISD::STRICT_FCOS:
2722  case ISD::STRICT_FEXP:
2723  case ISD::STRICT_FEXP2:
2724  case ISD::STRICT_FLOG:
2725  case ISD::STRICT_FLOG10:
2726  case ISD::STRICT_FLOG2:
2727  case ISD::STRICT_FRINT:
2729  case ISD::STRICT_FMAXNUM:
2730  case ISD::STRICT_FMINNUM:
2731  case ISD::STRICT_FCEIL:
2732  case ISD::STRICT_FFLOOR:
2733  case ISD::STRICT_FROUND:
2734  case ISD::STRICT_FTRUNC:
2735  Res = WidenVecRes_StrictFP(N);
2736  break;
2737 
2738  case ISD::UADDO:
2739  case ISD::SADDO:
2740  case ISD::USUBO:
2741  case ISD::SSUBO:
2742  case ISD::UMULO:
2743  case ISD::SMULO:
2744  Res = WidenVecRes_OverflowOp(N, ResNo);
2745  break;
2746 
2747  case ISD::FCOPYSIGN:
2748  Res = WidenVecRes_FCOPYSIGN(N);
2749  break;
2750 
2751  case ISD::FPOWI:
2752  Res = WidenVecRes_POWI(N);
2753  break;
2754 
2755  case ISD::SHL:
2756  case ISD::SRA:
2757  case ISD::SRL:
2758  Res = WidenVecRes_Shift(N);
2759  break;
2760 
2764  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2765  break;
2766 
2767  case ISD::ANY_EXTEND:
2768  case ISD::FP_EXTEND:
2769  case ISD::FP_ROUND:
2770  case ISD::FP_TO_SINT:
2771  case ISD::FP_TO_UINT:
2772  case ISD::SIGN_EXTEND:
2773  case ISD::SINT_TO_FP:
2774  case ISD::TRUNCATE:
2775  case ISD::UINT_TO_FP:
2776  case ISD::ZERO_EXTEND:
2777  Res = WidenVecRes_Convert(N);
2778  break;
2779 
2780  case ISD::STRICT_FP_EXTEND:
2781  case ISD::STRICT_FP_ROUND:
2782  Res = WidenVecRes_Convert_StrictFP(N);
2783  break;
2784 
2785  case ISD::FABS:
2786  case ISD::FCEIL:
2787  case ISD::FCOS:
2788  case ISD::FEXP:
2789  case ISD::FEXP2:
2790  case ISD::FFLOOR:
2791  case ISD::FLOG:
2792  case ISD::FLOG10:
2793  case ISD::FLOG2:
2794  case ISD::FNEARBYINT:
2795  case ISD::FRINT:
2796  case ISD::FROUND:
2797  case ISD::FSIN:
2798  case ISD::FSQRT:
2799  case ISD::FTRUNC: {
2800  // We're going to widen this vector op to a legal type by padding with undef
2801  // elements. If the wide vector op is eventually going to be expanded to
2802  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2803  // libcalls on the undef elements.
2804  EVT VT = N->getValueType(0);
2805  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2806  if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2807  TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
2808  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2809  break;
2810  }
2811  }
2812  // If the target has custom/legal support for the scalar FP intrinsic ops
2813  // (they are probably not destined to become libcalls), then widen those like
2814  // any other unary ops.
2816 
2817  case ISD::ABS:
2818  case ISD::BITREVERSE:
2819  case ISD::BSWAP:
2820  case ISD::CTLZ:
2821  case ISD::CTPOP:
2822  case ISD::CTTZ:
2823  case ISD::FNEG:
2824  case ISD::FCANONICALIZE:
2825  Res = WidenVecRes_Unary(N);
2826  break;
2827  case ISD::FMA:
2828  Res = WidenVecRes_Ternary(N);
2829  break;
2830  }
2831 
2832  // If Res is null, the sub-method took care of registering the result.
2833  if (Res.getNode())
2834  SetWidenedVector(SDValue(N, ResNo), Res);
2835 }
2836 
2837 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2838  // Ternary op widening.
2839  SDLoc dl(N);
2840  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2841  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2842  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2843  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2844  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2845 }
2846 
2847 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2848  // Binary op widening.
2849  SDLoc dl(N);
2850  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2851  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2852  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2853  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2854 }
2855 
2856 // Given a vector of operations that have been broken up to widen, see
2857 // if we can collect them together into the next widest legal VT. This
2858 // implementation is trap-safe.
2860  SmallVectorImpl<SDValue> &ConcatOps,
2861  unsigned ConcatEnd, EVT VT, EVT MaxVT,
2862  EVT WidenVT) {
2863  // Check to see if we have a single operation with the widen type.
2864  if (ConcatEnd == 1) {
2865  VT = ConcatOps[0].getValueType();
2866  if (VT == WidenVT)
2867  return ConcatOps[0];
2868  }
2869 
2870  SDLoc dl(ConcatOps[0]);
2871  EVT WidenEltVT = WidenVT.getVectorElementType();
2872 
2873  // while (Some element of ConcatOps is not of type MaxVT) {
2874  // From the end of ConcatOps, collect elements of the same type and put
2875  // them into an op of the next larger supported type
2876  // }
2877  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2878  int Idx = ConcatEnd - 1;
2879  VT = ConcatOps[Idx--].getValueType();
2880  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2881  Idx--;
2882 
2883  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2884  EVT NextVT;
2885  do {
2886  NextSize *= 2;
2887  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2888  } while (!TLI.isTypeLegal(NextVT));
2889 
2890  if (!VT.isVector()) {
2891  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2892  SDValue VecOp = DAG.getUNDEF(NextVT);
2893  unsigned NumToInsert = ConcatEnd - Idx - 1;
2894  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2895  VecOp = DAG.getNode(
2896  ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2897  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2898  }
2899  ConcatOps[Idx+1] = VecOp;
2900  ConcatEnd = Idx + 2;
2901  } else {
2902  // Vector type, create a CONCAT_VECTORS of type NextVT
2903  SDValue undefVec = DAG.getUNDEF(VT);
2904  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2905  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2906  unsigned RealVals = ConcatEnd - Idx - 1;
2907  unsigned SubConcatEnd = 0;
2908  unsigned SubConcatIdx = Idx + 1;
2909  while (SubConcatEnd < RealVals)
2910  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2911  while (SubConcatEnd < OpsToConcat)
2912  SubConcatOps[SubConcatEnd++] = undefVec;
2913  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2914  NextVT, SubConcatOps);
2915  ConcatEnd = SubConcatIdx + 1;
2916  }
2917  }
2918 
2919  // Check to see if we have a single operation with the widen type.
2920  if (ConcatEnd == 1) {
2921  VT = ConcatOps[0].getValueType();
2922  if (VT == WidenVT)
2923  return ConcatOps[0];
2924  }
2925 
2926  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2927  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2928  if (NumOps != ConcatEnd ) {
2929  SDValue UndefVal = DAG.getUNDEF(MaxVT);
2930  for (unsigned j = ConcatEnd; j < NumOps; ++j)
2931  ConcatOps[j] = UndefVal;
2932  }
2933  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2934  makeArrayRef(ConcatOps.data(), NumOps));
2935 }
2936 
2937 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2938  // Binary op widening for operations that can trap.
2939  unsigned Opcode = N->getOpcode();
2940  SDLoc dl(N);
2941  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2942  EVT WidenEltVT = WidenVT.getVectorElementType();
2943  EVT VT = WidenVT;
2944  unsigned NumElts = VT.getVectorNumElements();
2945  const SDNodeFlags Flags = N->getFlags();
2946  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2947  NumElts = NumElts / 2;
2948  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2949  }
2950 
2951  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2952  // Operation doesn't trap so just widen as normal.
2953  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2954  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2955  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2956  }
2957 
2958  // No legal vector version so unroll the vector operation and then widen.
2959  if (NumElts == 1)
2960  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2961 
2962  // Since the operation can trap, apply operation on the original vector.
2963  EVT MaxVT = VT;
2964  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2965  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2966  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2967 
2968  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2969  unsigned ConcatEnd = 0; // Current ConcatOps index.
2970  int Idx = 0; // Current Idx into input vectors.
2971 
2972  // NumElts := greatest legal vector size (at most WidenVT)
2973  // while (orig. vector has unhandled elements) {
2974  // take munches of size NumElts from the beginning and add to ConcatOps
2975  // NumElts := next smaller supported vector size or 1
2976  // }
2977  while (CurNumElts != 0) {
2978  while (CurNumElts >= NumElts) {
2979  SDValue EOp1 = DAG.getNode(
2980  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2981  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2982  SDValue EOp2 = DAG.getNode(
2983  ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2984  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2985  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2986  Idx += NumElts;
2987  CurNumElts -= NumElts;
2988  }
2989  do {
2990  NumElts = NumElts / 2;
2991  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2992  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2993 
2994  if (NumElts == 1) {
2995  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2996  SDValue EOp1 = DAG.getNode(
2997  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2998  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2999  SDValue EOp2 = DAG.getNode(
3000  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
3001  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3002  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3003  EOp1, EOp2, Flags);
3004  }
3005  CurNumElts = 0;
3006  }
3007  }
3008 
3009  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3010 }
3011 
3012 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3013  // StrictFP op widening for operations that can trap.
3014  unsigned NumOpers = N->getNumOperands();
3015  unsigned Opcode = N->getOpcode();
3016  SDLoc dl(N);
3017  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3018  EVT WidenEltVT = WidenVT.getVectorElementType();
3019  EVT VT = WidenVT;
3020  unsigned NumElts = VT.getVectorNumElements();
3021  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3022  NumElts = NumElts / 2;
3023  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3024  }
3025 
3026  // No legal vector version so unroll the vector operation and then widen.
3027  if (NumElts == 1)
3028  return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3029 
3030  // Since the operation can trap, apply operation on the original vector.
3031  EVT MaxVT = VT;
3033  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3034 
3035  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3036  SmallVector<SDValue, 16> Chains;
3037  unsigned ConcatEnd = 0; // Current ConcatOps index.
3038  int Idx = 0; // Current Idx into input vectors.
3039 
3040  // The Chain is the first operand.
3041  InOps.push_back(N->getOperand(0));
3042 
3043  // Now process the remaining operands.
3044  for (unsigned i = 1; i < NumOpers; ++i) {
3045  SDValue Oper = N->getOperand(i);
3046 
3047  if (Oper.getValueType().isVector()) {
3048  assert(Oper.getValueType() == N->getValueType(0) &&
3049  "Invalid operand type to widen!");
3050  Oper = GetWidenedVector(Oper);
3051  }
3052 
3053  InOps.push_back(Oper);
3054  }
3055 
3056  // NumElts := greatest legal vector size (at most WidenVT)
3057  // while (orig. vector has unhandled elements) {
3058  // take munches of size NumElts from the beginning and add to ConcatOps
3059  // NumElts := next smaller supported vector size or 1
3060  // }
3061  while (CurNumElts != 0) {
3062  while (CurNumElts >= NumElts) {
3064 
3065  for (unsigned i = 0; i < NumOpers; ++i) {
3066  SDValue Op = InOps[i];
3067 
3068  if (Op.getValueType().isVector())
3069  Op = DAG.getNode(
3070  ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3071  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3072 
3073  EOps.push_back(Op);
3074  }
3075 
3076  EVT OperVT[] = {VT, MVT::Other};
3077  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3078  ConcatOps[ConcatEnd++] = Oper;
3079  Chains.push_back(Oper.getValue(1));
3080  Idx += NumElts;
3081  CurNumElts -= NumElts;
3082  }
3083  do {
3084  NumElts = NumElts / 2;
3085  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3086  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3087 
3088  if (NumElts == 1) {
3089  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3091 
3092  for (unsigned i = 0; i < NumOpers; ++i) {
3093  SDValue Op = InOps[i];
3094 
3095  if (Op.getValueType().isVector())
3096  Op = DAG.getNode(
3097  ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3098  DAG.getConstant(Idx, dl,
3099  TLI.getVectorIdxTy(DAG.getDataLayout())));
3100 
3101  EOps.push_back(Op);
3102  }
3103 
3104  EVT WidenVT[] = {WidenEltVT, MVT::Other};
3105  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3106  ConcatOps[ConcatEnd++] = Oper;
3107  Chains.push_back(Oper.getValue(1));
3108  }
3109  CurNumElts = 0;
3110  }
3111  }
3112 
3113  // Build a factor node to remember all the Ops that have been created.
3114  SDValue NewChain;
3115  if (Chains.size() == 1)
3116  NewChain = Chains[0];
3117  else
3118  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3119  ReplaceValueWith(SDValue(N, 1), NewChain);
3120 
3121  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3122 }
3123 
3124 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3125  SDLoc DL(N);
3126  EVT ResVT = N->getValueType(0);
3127  EVT OvVT = N->getValueType(1);
3128  EVT WideResVT, WideOvVT;
3129  SDValue WideLHS, WideRHS;
3130 
3131  // TODO: This might result in a widen/split loop.
3132  if (ResNo == 0) {
3133  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3134  WideOvVT = EVT::getVectorVT(
3135  *DAG.getContext(), OvVT.getVectorElementType(),
3136  WideResVT.getVectorNumElements());
3137 
3138  WideLHS = GetWidenedVector(N->getOperand(0));
3139  WideRHS = GetWidenedVector(N->getOperand(1));
3140  } else {
3141  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3142  WideResVT = EVT::getVectorVT(
3143  *DAG.getContext(), ResVT.getVectorElementType(),
3144  WideOvVT.getVectorNumElements());
3145 
3146  SDValue Zero = DAG.getConstant(
3147  0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3148  WideLHS = DAG.getNode(
3149  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3150  N->getOperand(0), Zero);
3151  WideRHS = DAG.getNode(
3152  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3153  N->getOperand(1), Zero);
3154  }
3155 
3156  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3157  SDNode *WideNode = DAG.getNode(
3158  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3159 
3160  // Replace the other vector result not being explicitly widened here.
3161  unsigned OtherNo = 1 - ResNo;
3162  EVT OtherVT = N->getValueType(OtherNo);
3163  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3164  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3165  } else {
3166  SDValue Zero = DAG.getConstant(
3167  0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3168  SDValue OtherVal = DAG.getNode(
3169  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3170  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3171  }
3172 
3173  return SDValue(WideNode, ResNo);
3174 }
3175 
3176 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3177  SDValue InOp = N->getOperand(0);
3178  SDLoc DL(N);
3179 
3180  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3181  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3182 
3183  EVT InVT = InOp.getValueType();
3184  EVT InEltVT = InVT.getVectorElementType();
3185  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
3186 
3187  unsigned Opcode = N->getOpcode();
3188  unsigned InVTNumElts = InVT.getVectorNumElements();
3189  const SDNodeFlags Flags = N->getFlags();
3190  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3191  InOp = GetWidenedVector(N->getOperand(0));
3192  InVT = InOp.getValueType();
3193  InVTNumElts = InVT.getVectorNumElements();
3194  if (InVTNumElts == WidenNumElts) {
3195  if (N->getNumOperands() == 1)
3196  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3197  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3198  }
3199  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3200  // If both input and result vector types are of same width, extend
3201  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3202  // accepts fewer elements in the result than in the input.
3203  if (Opcode == ISD::ANY_EXTEND)
3204  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3205  if (Opcode == ISD::SIGN_EXTEND)
3206  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3207  if (Opcode == ISD::ZERO_EXTEND)
3208  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3209  }
3210  }
3211 
3212  if (TLI.isTypeLegal(InWidenVT)) {
3213  // Because the result and the input are different vector types, widening
3214  // the result could create a legal type but widening the input might make
3215  // it an illegal type that might lead to repeatedly splitting the input
3216  // and then widening it. To avoid this, we widen the input only if
3217  // it results in a legal type.
3218  if (WidenNumElts % InVTNumElts == 0) {
3219  // Widen the input and call convert on the widened input vector.
3220  unsigned NumConcat = WidenNumElts/InVTNumElts;
3221  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3222  Ops[0] = InOp;
3223  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3224  if (N->getNumOperands() == 1)
3225  return DAG.getNode(Opcode, DL, WidenVT, InVec);
3226  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3227  }
3228 
3229  if (InVTNumElts % WidenNumElts == 0) {
3230  SDValue InVal = DAG.getNode(
3231  ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3232  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3233  // Extract the input and convert the shorten input vector.
3234  if (N->getNumOperands() == 1)
3235  return DAG.getNode(Opcode, DL, WidenVT, InVal);
3236  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3237  }
3238  }
3239 
3240  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3241  EVT EltVT = WidenVT.getVectorElementType();
3242  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3243  // Use the original element count so we don't do more scalar opts than
3244  // necessary.
3245  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3246  for (unsigned i=0; i < MinElts; ++i) {
3247  SDValue Val = DAG.getNode(
3248  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3249  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3250  if (N->getNumOperands() == 1)
3251  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3252  else
3253  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3254  }
3255 
3256  return DAG.getBuildVector(WidenVT, DL, Ops);
3257 }
3258 
3259 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3260  SDValue InOp = N->getOperand(1);
3261  SDLoc DL(N);
3262  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3263 
3264  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3265  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3266  SmallVector<EVT, 2> WidenVTs = { WidenVT, MVT::Other };
3267 
3268  EVT InVT = InOp.getValueType();
3269  EVT InEltVT = InVT.getVectorElementType();
3270 
3271  unsigned Opcode = N->getOpcode();
3272 
3273  // FIXME: Optimizations need to be implemented here.
3274 
3275  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3276  EVT EltVT = WidenVT.getVectorElementType();
3277  SmallVector<EVT, 2> EltVTs = { EltVT, MVT::Other };
3278  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3279  SmallVector<SDValue, 32> OpChains;
3280  // Use the original element count so we don't do more scalar opts than
3281  // necessary.
3282  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3283  for (unsigned i=0; i < MinElts; ++i) {
3284  NewOps[1] = DAG.getNode(
3285  ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3286  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3287  Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3288  OpChains.push_back(Ops[i].getValue(1));
3289  }
3290  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3291  ReplaceValueWith(SDValue(N, 1), NewChain);
3292 
3293  return DAG.getBuildVector(WidenVT, DL, Ops);
3294 }
3295 
3296 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3297  unsigned Opcode = N->getOpcode();
3298  SDValue InOp = N->getOperand(0);
3299  SDLoc DL(N);
3300 
3301  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3302  EVT WidenSVT = WidenVT.getVectorElementType();
3303  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3304 
3305  EVT InVT = InOp.getValueType();
3306  EVT InSVT = InVT.getVectorElementType();
3307  unsigned InVTNumElts = InVT.getVectorNumElements();
3308 
3309  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3310  InOp = GetWidenedVector(InOp);
3311  InVT = InOp.getValueType();
3312  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3313  switch (Opcode) {
3317  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3318  }
3319  }
3320  }
3321 
3322  // Unroll, extend the scalars and rebuild the vector.
3324  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3325  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3326  DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3327  switch (Opcode) {
3329  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3330  break;
3332  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3333  break;
3335  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3336  break;
3337  default:
3338  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3339  }
3340  Ops.push_back(Val);
3341  }
3342 
3343  while (Ops.size() != WidenNumElts)
3344  Ops.push_back(DAG.getUNDEF(WidenSVT));
3345 
3346  return DAG.getBuildVector(WidenVT, DL, Ops);
3347 }
3348 
3349 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3350  // If this is an FCOPYSIGN with same input types, we can treat it as a
3351  // normal (can trap) binary op.
3352  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3353  return WidenVecRes_BinaryCanTrap(N);
3354 
3355  // If the types are different, fall back to unrolling.
3356  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3357  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3358 }
3359 
3360 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3361  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3362  SDValue InOp = GetWidenedVector(N->getOperand(0));
3363  SDValue ShOp = N->getOperand(1);
3364  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3365 }
3366 
3367 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
3368  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3369  SDValue InOp = GetWidenedVector(N->getOperand(0));
3370  SDValue ShOp = N->getOperand(1);
3371 
3372  EVT ShVT = ShOp.getValueType();
3373  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
3374  ShOp = GetWidenedVector(ShOp);
3375  ShVT = ShOp.getValueType();
3376  }
3377  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
3378  ShVT.getVectorElementType(),
3379  WidenVT.getVectorNumElements());
3380  if (ShVT != ShWidenVT)
3381  ShOp = ModifyToType(ShOp, ShWidenVT);
3382 
3383  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3384 }
3385 
3386 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3387  // Unary op widening.
3388  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3389  SDValue InOp = GetWidenedVector(N->getOperand(0));
3390  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3391 }
3392 
3393 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3394  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3395  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3396  cast<VTSDNode>(N->getOperand(1))->getVT()
3397  .getVectorElementType(),
3398  WidenVT.getVectorNumElements());
3399  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3400  return DAG.getNode(N->getOpcode(), SDLoc(N),
3401  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3402 }
3403 
3404 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3405  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3406  return GetWidenedVector(WidenVec);
3407 }
3408 
3409 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3410  SDValue InOp = N->getOperand(0);
3411  EVT InVT = InOp.getValueType();
3412  EVT VT = N->getValueType(0);
3413  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3414  SDLoc dl(N);
3415 
3416  switch (getTypeAction(InVT)) {
3418  break;
3420  // If the incoming type is a vector that is being promoted, then
3421  // we know that the elements are arranged differently and that we
3422  // must perform the conversion using a stack slot.
3423  if (InVT.isVector())
3424  break;
3425 
3426  // If the InOp is promoted to the same size, convert it. Otherwise,
3427  // fall out of the switch and widen the promoted input.
3428  InOp = GetPromotedInteger(InOp);
3429  InVT = InOp.getValueType();
3430  if (WidenVT.bitsEq(InVT))
3431  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3432  break;
3439  break;
3441  // If the InOp is widened to the same size, convert it. Otherwise, fall
3442  // out of the switch and widen the widened input.
3443  InOp = GetWidenedVector(InOp);
3444  InVT = InOp.getValueType();
3445  if (WidenVT.bitsEq(InVT))
3446  // The input widens to the same size. Convert to the widen value.
3447  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3448  break;
3449  }
3450 
3451  unsigned WidenSize = WidenVT.getSizeInBits();
3452  unsigned InSize = InVT.getSizeInBits();
3453  // x86mmx is not an acceptable vector element type, so don't try.
3454  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3455  // Determine new input vector type. The new input vector type will use
3456  // the same element type (if its a vector) or use the input type as a
3457  // vector. It is the same size as the type to widen to.
3458  EVT NewInVT;
3459  unsigned NewNumElts = WidenSize / InSize;
3460  if (InVT.isVector()) {
3461  EVT InEltVT = InVT.getVectorElementType();
3462  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3463  WidenSize / InEltVT.getSizeInBits());
3464  } else {
3465  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3466  }
3467 
3468  if (TLI.isTypeLegal(NewInVT)) {
3469  SDValue NewVec;
3470  if (InVT.isVector()) {
3471  // Because the result and the input are different vector types, widening
3472  // the result could create a legal type but widening the input might make
3473  // it an illegal type that might lead to repeatedly splitting the input
3474  // and then widening it. To avoid this, we widen the input only if
3475  // it results in a legal type.
3476  SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3477  Ops[0] = InOp;
3478 
3479  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3480  } else {
3481  NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3482  }
3483  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3484  }
3485  }
3486 
3487  return CreateStackStoreLoad(InOp, WidenVT);
3488 }
3489 
3490 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3491  SDLoc dl(N);
3492  // Build a vector with undefined for the new nodes.
3493  EVT VT = N->getValueType(0);
3494 
3495  // Integer BUILD_VECTOR operands may be larger than the node's vector element
3496  // type. The UNDEFs need to have the same type as the existing operands.
3497  EVT EltVT = N->getOperand(0).getValueType();
3498  unsigned NumElts = VT.getVectorNumElements();
3499 
3500  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3501  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3502 
3503  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3504  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3505  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3506 
3507  return DAG.getBuildVector(WidenVT, dl, NewOps);
3508 }
3509 
3510 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3511  EVT InVT = N->getOperand(0).getValueType();
3512  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3513  SDLoc dl(N);
3514  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3515  unsigned NumInElts = InVT.getVectorNumElements();
3516  unsigned NumOperands = N->getNumOperands();
3517 
3518  bool InputWidened = false; // Indicates we need to widen the input.
3519  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3520  if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
3521  // Add undef vectors to widen to correct length.
3522  unsigned NumConcat = WidenVT.getVectorNumElements() /
3523  InVT.getVectorNumElements();
3524  SDValue UndefVal = DAG.getUNDEF(InVT);
3525  SmallVector<SDValue, 16> Ops(NumConcat);
3526  for (unsigned i=0; i < NumOperands; ++i)
3527  Ops[i] = N->getOperand(i);
3528  for (unsigned i = NumOperands; i != NumConcat; ++i)
3529  Ops[i] = UndefVal;
3530  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3531  }
3532  } else {
3533  InputWidened = true;
3534  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3535  // The inputs and the result are widen to the same value.
3536  unsigned i;
3537  for (i=1; i < NumOperands; ++i)
3538  if (!N->getOperand(i).isUndef())
3539  break;
3540 
3541  if (i == NumOperands)
3542  // Everything but the first operand is an UNDEF so just return the
3543  // widened first operand.
3544  return GetWidenedVector(N->getOperand(0));
3545 
3546  if (NumOperands == 2) {
3547  // Replace concat of two operands with a shuffle.
3548  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3549  for (unsigned i = 0; i < NumInElts; ++i) {
3550  MaskOps[i] = i;
3551  MaskOps[i + NumInElts] = i + WidenNumElts;
3552  }
3553  return DAG.getVectorShuffle(WidenVT, dl,
3554  GetWidenedVector(N->getOperand(0)),
3555  GetWidenedVector(N->getOperand(1)),
3556  MaskOps);
3557  }
3558  }
3559  }
3560 
3561  // Fall back to use extracts and build vector.
3562  EVT EltVT = WidenVT.getVectorElementType();
3563  SmallVector<SDValue, 16> Ops(WidenNumElts);
3564  unsigned Idx = 0;
3565  for (unsigned i=0; i < NumOperands; ++i) {
3566  SDValue InOp = N->getOperand(i);
3567  if (InputWidened)
3568  InOp = GetWidenedVector(InOp);
3569  for (unsigned j=0; j < NumInElts; ++j)
3570  Ops[Idx++] = DAG.getNode(
3571  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3572  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3573  }
3574  SDValue UndefVal = DAG.getUNDEF(EltVT);
3575  for (; Idx < WidenNumElts; ++Idx)
3576  Ops[Idx] = UndefVal;
3577  return DAG.getBuildVector(WidenVT, dl, Ops);
3578 }
3579 
3580 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3581  EVT VT = N->getValueType(0);
3582  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3583  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3584  SDValue InOp = N->getOperand(0);
3585  SDValue Idx = N->getOperand(1);
3586  SDLoc dl(N);
3587 
3588  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3589  InOp = GetWidenedVector(InOp);
3590 
3591  EVT InVT = InOp.getValueType();
3592 
3593  // Check if we can just return the input vector after widening.
3594  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3595  if (IdxVal == 0 && InVT == WidenVT)
3596  return InOp;
3597 
3598  // Check if we can extract from the vector.
3599  unsigned InNumElts = InVT.getVectorNumElements();
3600  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3601  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3602 
3603  // We could try widening the input to the right length but for now, extract
3604  // the original elements, fill the rest with undefs and build a vector.
3605  SmallVector<SDValue, 16> Ops(WidenNumElts);
3606  EVT EltVT = VT.getVectorElementType();
3607  unsigned NumElts = VT.getVectorNumElements();
3608  unsigned i;
3609  for (i=0; i < NumElts; ++i)
3610  Ops[i] =
3611  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3612  DAG.getConstant(IdxVal + i, dl,
3613  TLI.getVectorIdxTy(DAG.getDataLayout())));
3614 
3615  SDValue UndefVal = DAG.getUNDEF(EltVT);
3616  for (; i < WidenNumElts; ++i)
3617  Ops[i] = UndefVal;
3618  return DAG.getBuildVector(WidenVT, dl, Ops);
3619 }
3620 
3621 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3622  SDValue InOp = GetWidenedVector(N->getOperand(0));
3623  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3624  InOp.getValueType(), InOp,
3625  N->getOperand(1), N->getOperand(2));
3626 }
3627 
3628 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3629  LoadSDNode *LD = cast<LoadSDNode>(N);
3631 
3632  SDValue Result;
3633  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3634  if (ExtType != ISD::NON_EXTLOAD)
3635  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3636  else
3637  Result = GenWidenVectorLoads(LdChain, LD);
3638 
3639  // If we generate a single load, we can use that for the chain. Otherwise,
3640  // build a factor node to remember the multiple loads are independent and
3641  // chain to that.
3642  SDValue NewChain;
3643  if (LdChain.size() == 1)
3644  NewChain = LdChain[0];
3645  else
3646  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3647 
3648  // Modified the chain - switch anything that used the old chain to use
3649  // the new one.
3650  ReplaceValueWith(SDValue(N, 1), NewChain);
3651 
3652  return Result;
3653 }
3654 
3655 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3656 
3657  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3658  SDValue Mask = N->getMask();
3659  EVT MaskVT = Mask.getValueType();
3660  SDValue PassThru = GetWidenedVector(N->getPassThru());
3662  SDLoc dl(N);
3663 
3664  // The mask should be widened as well
3665  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3666  MaskVT.getVectorElementType(),
3667  WidenVT.getVectorNumElements());
3668  Mask = ModifyToType(Mask, WideMaskVT, true);
3669 
3670  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3671  Mask, PassThru, N->getMemoryVT(),
3672  N->getMemOperand(), ExtType,
3673  N->isExpandingLoad());
3674  // Legalize the chain result - switch anything that used the old chain to
3675  // use the new one.
3676  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3677  return Res;
3678 }
3679 
3680 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3681 
3682  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3683  SDValue Mask = N->getMask();
3684  EVT MaskVT = Mask.getValueType();
3685  SDValue PassThru = GetWidenedVector(N->getPassThru());
3686  SDValue Scale = N->getScale();
3687  unsigned NumElts = WideVT.getVectorNumElements();
3688  SDLoc dl(N);
3689 
3690  // The mask should be widened as well
3691  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3692  MaskVT.getVectorElementType(),
3693  WideVT.getVectorNumElements());
3694  Mask = ModifyToType(Mask, WideMaskVT, true);
3695 
3696  // Widen the Index operand
3697  SDValue Index = N->getIndex();
3698  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3699  Index.getValueType().getScalarType(),
3700  NumElts);
3701  Index = ModifyToType(Index, WideIndexVT);
3702  SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3703  Scale };
3704  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3705  N->getMemoryVT(), dl, Ops,
3706  N->getMemOperand());
3707 
3708  // Legalize the chain result - switch anything that used the old chain to
3709  // use the new one.
3710  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3711  return Res;
3712 }
3713 
3714 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3715  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3716  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3717  WidenVT, N->getOperand(0));
3718 }
3719 
3720 // Return true if this is a node that could have two SETCCs as operands.
3721 static inline bool isLogicalMaskOp(unsigned Opcode) {
3722  switch (Opcode) {
3723  case ISD::AND:
3724  case ISD::OR:
3725  case ISD::XOR:
3726  return true;
3727  }
3728  return false;
3729 }
3730 
3731 // This is used just for the assert in convertMask(). Check that this either
3732 // a SETCC or a previously handled SETCC by convertMask().
3733 #ifndef NDEBUG
3734 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3735  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3736  N = N.getOperand(0);
3737  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3738  for (unsigned i = 1; i < N->getNumOperands(); ++i)
3739  if (!N->getOperand(i)->isUndef())
3740  return false;
3741  N = N.getOperand(0);
3742  }
3743 
3744  if (N.getOpcode() == ISD::TRUNCATE)
3745  N = N.getOperand(0);
3746  else if (N.getOpcode() == ISD::SIGN_EXTEND)
3747  N = N.getOperand(0);
3748 
3749  if (isLogicalMaskOp(N.getOpcode()))
3750  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3752 
3753  return (N.getOpcode() == ISD::SETCC ||
3755 }
3756 #endif
3757 
3758 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3759 // to ToMaskVT if needed with vector extension or truncation.
3760 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3761  EVT ToMaskVT) {
3762  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3763  // FIXME: This code seems to be too restrictive, we might consider
3764  // generalizing it or dropping it.
3765  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3766 
3767  // Make a new Mask node, with a legal result VT.
3769  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3770  Ops.push_back(InMask->getOperand(i));
3771  SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3772 
3773  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3774  // extend or truncate is needed.
3775  LLVMContext &Ctx = *DAG.getContext();
3776  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3777  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3778  if (MaskScalarBits < ToMaskScalBits) {
3779  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3780  MaskVT.getVectorNumElements());
3781  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3782  } else if (MaskScalarBits > ToMaskScalBits) {
3783  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3784  MaskVT.getVectorNumElements());
3785  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3786  }
3787 
3789  ToMaskVT.getScalarSizeInBits() &&
3790  "Mask should have the right element size by now.");
3791 
3792  // Adjust Mask to the right number of elements.
3793  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3794  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3795  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3796  SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3797  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3798  ZeroIdx);
3799  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3800  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3801  EVT SubVT = Mask->getValueType(0);
3802  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3803  SubOps[0] = Mask;
3804  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3805  }
3806 
3807  assert((Mask->getValueType(0) == ToMaskVT) &&
3808  "A mask of ToMaskVT should have been produced by now.");
3809 
3810  return Mask;
3811 }
3812 
3813 // This method tries to handle VSELECT and its mask by legalizing operands
3814 // (which may require widening) and if needed adjusting the mask vector type
3815 // to match that of the VSELECT. Without it, many cases end up with
3816 // scalarization of the SETCC, with many unnecessary instructions.
3817 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3818  LLVMContext &Ctx = *DAG.getContext();
3819  SDValue Cond = N->getOperand(0);
3820 
3821  if (N->getOpcode() != ISD::VSELECT)
3822  return SDValue();
3823 
3824  if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3825  return SDValue();
3826 
3827  // If this is a splitted VSELECT that was previously already handled, do
3828  // nothing.
3829  EVT CondVT = Cond->getValueType(0);
3830  if (CondVT.getScalarSizeInBits() != 1)
3831  return SDValue();
3832 
3833  EVT VSelVT = N->getValueType(0);
3834  // Only handle vector types which are a power of 2.
3835  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3836  return SDValue();
3837 
3838  // Don't touch if this will be scalarized.
3839  EVT FinalVT = VSelVT;
3840  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3841  FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3842 
3843  if (FinalVT.getVectorNumElements() == 1)
3844  return SDValue();
3845 
3846  // If there is support for an i1 vector mask, don't touch.
3847  if (Cond.getOpcode() == ISD::SETCC) {
3848  EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3849  while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3850  SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3851  EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3852  if (SetCCResVT.getScalarSizeInBits() == 1)
3853  return SDValue();
3854  } else if (CondVT.getScalarType() == MVT::i1) {
3855  // If there is support for an i1 vector mask (or only scalar i1 conditions),
3856  // don't touch.
3857  while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3858  CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3859 
3860  if (CondVT.getScalarType() == MVT::i1)
3861  return SDValue();
3862  }
3863 
3864  // Get the VT and operands for VSELECT, and widen if needed.
3865  SDValue VSelOp1 = N->getOperand(1);
3866  SDValue VSelOp2 = N->getOperand(2);
3867  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3868  VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3869  VSelOp1 = GetWidenedVector(VSelOp1);
3870  VSelOp2 = GetWidenedVector(VSelOp2);
3871  }
3872 
3873  // The mask of the VSELECT should have integer elements.
3874  EVT ToMaskVT = VSelVT;
3875  if (!ToMaskVT.getScalarType().isInteger())
3876  ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3877 
3878  SDValue Mask;
3879  if (Cond->getOpcode() == ISD::SETCC) {
3880  EVT MaskVT = getSetCCResultType(Cond.getOperand(0).getValueType());
3881  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3882  } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3883  Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3884  Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3885  // Cond is (AND/OR/XOR (SETCC, SETCC))
3886  SDValue SETCC0 = Cond->getOperand(0);
3887  SDValue SETCC1 = Cond->getOperand(1);
3888  EVT VT0 = getSetCCResultType(SETCC0.getOperand(0).getValueType());
3889  EVT VT1 = getSetCCResultType(SETCC1.getOperand(0).getValueType());
3890  unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3891  unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3892  unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3893  EVT MaskVT;
3894  // If the two SETCCs have different VTs, either extend/truncate one of
3895  // them to the other "towards" ToMaskVT, or truncate one and extend the
3896  // other to ToMaskVT.
3897  if (ScalarBits0 != ScalarBits1) {
3898  EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3899  EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3900  if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3901  MaskVT = WideVT;
3902  else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3903  MaskVT = NarrowVT;
3904  else
3905  MaskVT = ToMaskVT;
3906  } else
3907  // If the two SETCCs have the same VT, don't change it.
3908  MaskVT = VT0;
3909 
3910  // Make new SETCCs and logical nodes.
3911  SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3912  SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3913  Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3914 
3915  // Convert the logical op for VSELECT if needed.
3916  Mask = convertMask(Cond, MaskVT, ToMaskVT);
3917  } else
3918  return SDValue();
3919 
3920  return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3921 }
3922 
3923 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3924  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3925  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3926 
3927  SDValue Cond1 = N->getOperand(0);
3928  EVT CondVT = Cond1.getValueType();
3929  if (CondVT.isVector()) {
3930  if (SDValue Res = WidenVSELECTAndMask(N))
3931  return Res;
3932 
3933  EVT CondEltVT = CondVT.getVectorElementType();
3934  EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3935  CondEltVT, WidenNumElts);
3936  if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3937  Cond1 = GetWidenedVector(Cond1);
3938 
3939  // If we have to split the condition there is no point in widening the
3940  // select. This would result in an cycle of widening the select ->
3941  // widening the condition operand -> splitting the condition operand ->
3942  // splitting the select -> widening the select. Instead split this select
3943  // further and widen the resulting type.
3944  if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3945  SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3946  SDValue Res = ModifyToType(SplitSelect, WidenVT);
3947  return Res;
3948  }
3949 
3950  if (Cond1.getValueType() != CondWidenVT)
3951  Cond1 = ModifyToType(Cond1, CondWidenVT);
3952  }
3953 
3954  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3955  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3956  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3957  return DAG.getNode(N->getOpcode(), SDLoc(N),
3958  WidenVT, Cond1, InOp1, InOp2);
3959 }
3960 
3961 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3962  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3963  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3964  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3965  InOp1.getValueType(), N->getOperand(0),
3966  N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3967 }
3968 
3969 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3970  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3971  return DAG.getUNDEF(WidenVT);
3972 }
3973 
3974 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3975  EVT VT = N->getValueType(0);
3976  SDLoc dl(N);
3977 
3978  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3979  unsigned NumElts = VT.getVectorNumElements();
3980  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3981 
3982  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3983  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3984 
3985  // Adjust mask based on new input vector length.
3986  SmallVector<int, 16> NewMask;
3987  for (unsigned i = 0; i != NumElts; ++i) {
3988  int Idx = N->getMaskElt(i);
3989  if (Idx < (int)NumElts)
3990  NewMask.push_back(Idx);
3991  else
3992  NewMask.push_back(Idx - NumElts + WidenNumElts);
3993  }
3994  for (unsigned i = NumElts; i != WidenNumElts; ++i)
3995  NewMask.push_back(-1);
3996  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
3997 }
3998 
3999 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4000  assert(N->getValueType(0).isVector() &&
4001  N->getOperand(0).getValueType().isVector() &&
4002  "Operands must be vectors");
4003  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4004  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4005 
4006  SDValue InOp1 = N->getOperand(0);
4007  EVT InVT = InOp1.getValueType();
4008  assert(InVT.isVector() && "can not widen non-vector type");
4009  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4010  InVT.getVectorElementType(), WidenNumElts);
4011 
4012  // The input and output types often differ here, and it could be that while
4013  // we'd prefer to widen the result type, the input operands have been split.
4014  // In this case, we also need to split the result of this node as well.
4015  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
4016  SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4017  SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4018  return Res;
4019  }
4020 
4021  // If the inputs also widen, handle them directly. Otherwise widen by hand.
4022  SDValue InOp2 = N->getOperand(1);
4023  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4024  InOp1 = GetWidenedVector(InOp1);
4025  InOp2 = GetWidenedVector(InOp2);
4026  } else {
4027  InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4028  InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4029  }
4030 
4031  // Assume that the input and output will be widen appropriately. If not,
4032  // we will have to unroll it at some point.
4033  assert(InOp1.getValueType() == WidenInVT &&
4034  InOp2.getValueType() == WidenInVT &&
4035  "Input not widened to expected type!");
4036  (void)WidenInVT;
4037  return DAG.getNode(ISD::SETCC, SDLoc(N),
4038  WidenVT, InOp1, InOp2, N->getOperand(2));
4039 }
4040 
4041 
4042 //===----------------------------------------------------------------------===//
4043 // Widen Vector Operand
4044 //===----------------------------------------------------------------------===//
4045 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
4046  LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4047  dbgs() << "\n");
4048  SDValue Res = SDValue();
4049 
4050  // See if the target wants to custom widen this node.
4051  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4052  return false;
4053 
4054  switch (N->getOpcode()) {
4055  default:
4056 #ifndef NDEBUG
4057  dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
4058  N->dump(&DAG);
4059  dbgs() << "\n";
4060 #endif
4061  llvm_unreachable("Do not know how to widen this operator's operand!");
4062 
4063  case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
4064  case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
4065  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
4066  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
4067  case ISD::STORE: Res = WidenVecOp_STORE(N); break;
4068  case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
4069  case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
4070  case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
4071  case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
4072  case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
4073  case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
4074 
4075  case ISD::ANY_EXTEND:
4076  case ISD::SIGN_EXTEND:
4077  case ISD::ZERO_EXTEND:
4078  Res = WidenVecOp_EXTEND(N);
4079  break;
4080 
4081  case ISD::FP_EXTEND:
4082  case ISD::STRICT_FP_EXTEND:
4083  case ISD::FP_TO_SINT:
4084  case ISD::FP_TO_UINT:
4085  case ISD::SINT_TO_FP:
4086  case ISD::UINT_TO_FP:
4087  case ISD::TRUNCATE:
4088  Res = WidenVecOp_Convert(N);
4089  break;
4090 
4091  case ISD::VECREDUCE_FADD:
4092  case ISD::VECREDUCE_FMUL:
4093  case ISD::VECREDUCE_ADD:
4094  case ISD::VECREDUCE_MUL:
4095  case ISD::VECREDUCE_AND:
4096  case ISD::VECREDUCE_OR:
4097  case ISD::VECREDUCE_XOR:
4098  case ISD::VECREDUCE_SMAX:
4099  case ISD::VECREDUCE_SMIN:
4100  case ISD::VECREDUCE_UMAX:
4101  case ISD::VECREDUCE_UMIN:
4102  case ISD::VECREDUCE_FMAX:
4103  case ISD::VECREDUCE_FMIN:
4104  Res = WidenVecOp_VECREDUCE(N);
4105  break;
4106  }
4107 
4108  // If Res is null, the sub-method took care of registering the result.
4109  if (!Res.getNode()) return false;
4110 
4111  // If the result is N, the sub-method updated N in place. Tell the legalizer
4112  // core about this.
4113  if (Res.getNode() == N)
4114  return true;
4115 
4116 
4117  if (N->isStrictFPOpcode())
4118  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4119  "Invalid operand expansion");
4120  else
4121  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4122  "Invalid operand expansion");
4123 
4124  ReplaceValueWith(SDValue(N, 0), Res);
4125  return false;
4126 }
4127 
4128 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4129  SDLoc DL(N);
4130  EVT VT = N->getValueType(0);
4131 
4132  SDValue InOp = N->getOperand(0);
4133  assert(getTypeAction(InOp.getValueType()) ==
4135  "Unexpected type action");
4136  InOp = GetWidenedVector(InOp);
4139  "Input wasn't widened!");
4140 
4141  // We may need to further widen the operand until it has the same total
4142  // vector size as the result.
4143  EVT InVT = InOp.getValueType();
4144  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
4145  EVT InEltVT = InVT.getVectorElementType();
4146  for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
4147  EVT FixedVT = (MVT::SimpleValueType)i;
4148  EVT FixedEltVT = FixedVT.getVectorElementType();
4149  if (TLI.isTypeLegal(FixedVT) &&
4150  FixedVT.getSizeInBits() == VT.getSizeInBits() &&
4151  FixedEltVT == InEltVT) {
4152  assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
4153  "Not enough elements in the fixed type for the operand!");
4154  assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
4155  "We can't have the same type as we started with!");
4156  if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
4157  InOp = DAG.getNode(
4158  ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
4159  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4160  else
4161  InOp = DAG.getNode(
4162  ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
4163  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4164  break;
4165  }
4166  }
4167  InVT = InOp.getValueType();
4168  if (InVT.getSizeInBits() != VT.getSizeInBits())
4169  // We couldn't find a legal vector type that was a widening of the input
4170  // and could be extended in-register to the result type, so we have to
4171  // scalarize.
4172  return WidenVecOp_Convert(N);
4173  }
4174 
4175  // Use special DAG nodes to represent the operation of extending the
4176  // low lanes.
4177  switch (N->getOpcode()) {
4178  default:
4179  llvm_unreachable("Extend legalization on extend operation!");
4180  case ISD::ANY_EXTEND:
4181  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4182  case ISD::SIGN_EXTEND:
4183  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4184  case ISD::ZERO_EXTEND:
4185  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4186  }
4187 }
4188 
4189 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4190  // The result (and first input) is legal, but the second input is illegal.
4191  // We can't do much to fix that, so just unroll and let the extracts off of
4192  // the second input be widened as needed later.
4193  return DAG.UnrollVectorOp(N);
4194 }
4195 
4196 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4197  // Since the result is legal and the input is illegal.
4198  EVT VT = N->getValueType(0);
4199  EVT EltVT = VT.getVectorElementType();
4200  SDLoc dl(N);
4201  unsigned NumElts = VT.getVectorNumElements();
4202  SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4203  assert(getTypeAction(InOp.getValueType()) ==
4205  "Unexpected type action");
4206  InOp = GetWidenedVector(InOp);
4207  EVT InVT = InOp.getValueType();
4208  unsigned Opcode = N->getOpcode();
4209 
4210  // See if a widened result type would be legal, if so widen the node.
4211  // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
4212  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4213  InVT.getVectorNumElements());
4214  if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
4215  SDValue Res;
4216  if (N->isStrictFPOpcode()) {
4217  Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4218  { N->getOperand(0), InOp });
4219  // Legalize the chain result - switch anything that used the old chain to
4220  // use the new one.
4221  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4222  } else
4223  Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4224  return DAG.getNode(
4225  ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
4226  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4227  }
4228 
4229  EVT InEltVT = InVT.getVectorElementType();
4230 
4231  // Unroll the convert into some scalar code and create a nasty build vector.
4232  SmallVector<SDValue, 16> Ops(NumElts);
4233  if (N->isStrictFPOpcode()) {
4234  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4235  SmallVector<SDValue, 32> OpChains;
4236  for (unsigned i=0; i < NumElts; ++i) {
4237  NewOps[1] = DAG.getNode(
4238  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4239  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4240  Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4241  OpChains.push_back(Ops[i].getValue(1));
4242  }
4243  SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4244  ReplaceValueWith(SDValue(N, 1), NewChain);
4245  } else {
4246  for (unsigned i = 0; i < NumElts; ++i)
4247  Ops[i] = DAG.getNode(
4248  Opcode, dl, EltVT,
4249  DAG.getNode(
4250  ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4251  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
4252  }
4253 
4254  return DAG.getBuildVector(VT, dl, Ops);
4255 }
4256 
4257 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4258  EVT VT = N->getValueType(0);
4259  SDValue InOp = GetWidenedVector(N->getOperand(0));
4260  EVT InWidenVT = InOp.getValueType();
4261  SDLoc dl(N);
4262 
4263  // Check if we can convert between two legal vector types and extract.
4264  unsigned InWidenSize = InWidenVT.getSizeInBits();
4265  unsigned Size = VT.getSizeInBits();
4266  // x86mmx is not an acceptable vector element type, so don't try.
4267  if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
4268  unsigned NewNumElts = InWidenSize / Size;
4269  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4270  if (TLI.isTypeLegal(NewVT)) {
4271  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4272  return DAG.getNode(
4273  ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4274  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4275  }
4276  }
4277 
4278  // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
4279  // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
4280  // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
4281  // having to copy via memory.
4282  if (VT.isVector()) {
4283  EVT EltVT = VT.getVectorElementType();
4284  unsigned EltSize = EltVT.getSizeInBits();
4285  if (InWidenSize % EltSize == 0) {
4286  unsigned NewNumElts = InWidenSize / EltSize;
4287  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4288  if (TLI.isTypeLegal(NewVT)) {
4289  SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4290  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4291  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4292  }
4293  }
4294  }
4295 
4296  return CreateStackStoreLoad(InOp, VT);
4297 }
4298 
4299 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4300  EVT VT = N->getValueType(0);
4301  EVT EltVT = VT.getVectorElementType();
4302  EVT InVT = N->getOperand(0).getValueType();
4303  SDLoc dl(N);
4304 
4305  // If the widen width for this operand is the same as the width of the concat
4306  // and all but the first operand is undef, just use the widened operand.
4307  unsigned NumOperands = N->getNumOperands();
4308  if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4309  unsigned i;
4310  for (i = 1; i < NumOperands; ++i)
4311  if (!N->getOperand(i).isUndef())
4312  break;
4313 
4314  if (i == NumOperands)
4315  return GetWidenedVector(N->getOperand(0));
4316  }
4317 
4318  // Otherwise, fall back to a nasty build vector.
4319  unsigned NumElts = VT.getVectorNumElements();
4320  SmallVector<SDValue, 16> Ops(NumElts);
4321 
4322  unsigned NumInElts = InVT.getVectorNumElements();
4323 
4324  unsigned Idx = 0;
4325  for (unsigned i=0; i < NumOperands; ++i) {
4326  SDValue InOp = N->getOperand(i);
4327  assert(getTypeAction(InOp.getValueType()) ==
4329  "Unexpected type action");
4330  InOp = GetWidenedVector(InOp);
4331  for (unsigned j=0; j < NumInElts; ++j)
4332  Ops[Idx++] = DAG.getNode(
4333  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4334  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4335  }
4336  return DAG.getBuildVector(VT, dl, Ops);
4337 }
4338 
4339 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4340  SDValue InOp = GetWidenedVector(N->getOperand(0));
4341  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4342  N->getValueType(0), InOp, N->getOperand(1));
4343 }
4344 
4345 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4346  SDValue InOp = GetWidenedVector(N->getOperand(0));
4347  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4348  N->getValueType(0), InOp, N->getOperand(1));
4349 }
4350 
4351 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4352  // We have to widen the value, but we want only to store the original
4353  // vector type.
4354  StoreSDNode *ST = cast<StoreSDNode>(N);
4355 
4356  if (!ST->getMemoryVT().getScalarType().isByteSized())
4357  return TLI.scalarizeVectorStore(ST, DAG);
4358 
4359  SmallVector<SDValue, 16> StChain;
4360  if (ST->isTruncatingStore())
4361  GenWidenVectorTruncStores(StChain, ST);
4362  else
4363  GenWidenVectorStores(StChain, ST);
4364 
4365  if (StChain.size() == 1)
4366  return StChain[0];
4367  else
4368  return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4369 }
4370 
4371 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4372  assert((OpNo == 1 || OpNo == 3) &&
4373  "Can widen only data or mask operand of mstore");
4374  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4375  SDValue Mask = MST->getMask();
4376  EVT MaskVT = Mask.getValueType();
4377  SDValue StVal = MST->getValue();
4378  SDLoc dl(N);
4379 
4380  if (OpNo == 1) {
4381  // Widen the value.
4382  StVal = GetWidenedVector(StVal);
4383 
4384  // The mask should be widened as well.
4385  EVT WideVT = StVal.getValueType();
4386  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4387  MaskVT.getVectorElementType(),
4388  WideVT.getVectorNumElements());
4389  Mask = ModifyToType(Mask, WideMaskVT, true);
4390  } else {
4391  // Widen the mask.
4392  EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4393  Mask = ModifyToType(Mask, WideMaskVT, true);
4394 
4395  EVT ValueVT = StVal.getValueType();
4396  EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4397  ValueVT.getVectorElementType(),
4398  WideMaskVT.getVectorNumElements());
4399  StVal = ModifyToType(StVal, WideVT);
4400  }
4401 
4402  assert(Mask.getValueType().getVectorNumElements() ==
4403  StVal.getValueType().getVectorNumElements() &&
4404  "Mask and data vectors should have the same number of elements");
4405  return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4406  Mask, MST->getMemoryVT(), MST->getMemOperand(),
4407  false, MST->isCompressingStore());
4408 }
4409 
4410 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4411  assert(OpNo == 4 && "Can widen only the index of mgather");
4412  auto *MG = cast<MaskedGatherSDNode>(N);
4413  SDValue DataOp = MG->getPassThru();
4414  SDValue Mask = MG->getMask();
4415  SDValue Scale = MG->getScale();
4416 
4417  // Just widen the index. It's allowed to have extra elements.
4418  SDValue Index = GetWidenedVector(MG->getIndex());
4419 
4420  SDLoc dl(N);
4421  SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4422  Scale};
4423  SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4424  MG->getMemOperand());
4425  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4426  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4427  return SDValue();
4428 }
4429 
4430 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4431  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4432  SDValue DataOp = MSC->getValue();
4433  SDValue Mask = MSC->getMask();
4434  SDValue Index = MSC->getIndex();
4435  SDValue Scale = MSC->getScale();
4436 
4437  if (OpNo == 1) {
4438  DataOp = GetWidenedVector(DataOp);
4439  unsigned NumElts = DataOp.getValueType().getVectorNumElements();
4440 
4441  // Widen index.
4442  EVT IndexVT = Index.getValueType();
4443  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4444  IndexVT.getVectorElementType(), NumElts);
4445  Index = ModifyToType(Index, WideIndexVT);
4446 
4447  // The mask should be widened as well.
4448  EVT MaskVT = Mask.getValueType();
4449  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4450  MaskVT.getVectorElementType(), NumElts);
4451  Mask = ModifyToType(Mask, WideMaskVT, true);
4452  } else if (OpNo == 4) {
4453  // Just widen the index. It's allowed to have extra elements.
4454  Index = GetWidenedVector(Index);
4455  } else
4456  llvm_unreachable("Can't widen this operand of mscatter");
4457 
4458  SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4459  Scale};
4460  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4461  MSC->getMemoryVT(), SDLoc(N), Ops,
4462  MSC->getMemOperand());
4463 }
4464 
4465 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4466  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4467  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4468  SDLoc dl(N);
4469  EVT VT = N->getValueType(0);
4470 
4471  // WARNING: In this code we widen the compare instruction with garbage.
4472  // This garbage may contain denormal floats which may be slow. Is this a real
4473  // concern ? Should we zero the unused lanes if this is a float compare ?
4474 
4475  // Get a new SETCC node to compare the newly widened operands.
4476  // Only some of the compared elements are legal.
4477  EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
4478  InOp0.getValueType());
4479  // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
4480  if (VT.getScalarType() == MVT::i1)
4481  SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4482  SVT.getVectorNumElements());
4483 
4484  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4485  SVT, InOp0, InOp1, N->getOperand(2));
4486 
4487  // Extract the needed results from the result vector.
4488  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
4489  SVT.getVectorElementType(),
4490  VT.getVectorNumElements());
4491  SDValue CC = DAG.getNode(
4492  ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
4493  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4494 
4495  return PromoteTargetBoolean(CC, VT);
4496 }
4497 
4498 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
4499  SDLoc dl(N);
4500  SDValue Op = GetWidenedVector(N->getOperand(0));
4501  EVT OrigVT = N->getOperand(0).getValueType();
4502  EVT WideVT = Op.getValueType();
4503  EVT ElemVT = OrigVT.getVectorElementType();
4504 
4505  SDValue NeutralElem;
4506  switch (N->getOpcode()) {
4507  case ISD::VECREDUCE_ADD:
4508  case ISD::VECREDUCE_OR:
4509  case ISD::VECREDUCE_XOR:
4510  case ISD::VECREDUCE_UMAX:
4511  NeutralElem = DAG.getConstant(0, dl, ElemVT);
4512  break;
4513  case ISD::VECREDUCE_MUL:
4514  NeutralElem = DAG.getConstant(1, dl, ElemVT);
4515  break;
4516  case ISD::VECREDUCE_AND:
4517  case ISD::VECREDUCE_UMIN:
4518  NeutralElem = DAG.getAllOnesConstant(dl, ElemVT);
4519  break;
4520  case ISD::VECREDUCE_SMAX:
4521  NeutralElem = DAG.getConstant(
4522  APInt::getSignedMinValue(ElemVT.getSizeInBits()), dl, ElemVT);
4523  break;
4524  case ISD::VECREDUCE_SMIN:
4525  NeutralElem = DAG.getConstant(
4526  APInt::getSignedMaxValue(ElemVT.getSizeInBits()), dl, ElemVT);
4527  break;
4528  case ISD::VECREDUCE_FADD:
4529  NeutralElem = DAG.getConstantFP(0.0, dl, ElemVT);
4530  break;
4531  case ISD::VECREDUCE_FMUL:
4532  NeutralElem = DAG.getConstantFP(1.0, dl, ElemVT);
4533  break;
4534  case ISD::VECREDUCE_FMAX:
4535  NeutralElem = DAG.getConstantFP(
4536  std::numeric_limits<double>::infinity(), dl, ElemVT);
4537  break;
4538  case ISD::VECREDUCE_FMIN:
4539  NeutralElem = DAG.getConstantFP(
4540  -std::numeric_limits<double>::infinity(), dl, ElemVT);
4541  break;
4542  }
4543 
4544  // Pad the vector with the neutral element.
4545  unsigned OrigElts = OrigVT.getVectorNumElements();
4546  unsigned WideElts = WideVT.getVectorNumElements();
4547  for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
4548  Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
4549  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4550 
4551  return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags());
4552 }
4553 
4554 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
4555  // This only gets called in the case that the left and right inputs and
4556  // result are of a legal odd vector type, and the condition is illegal i1 of
4557  // the same odd width that needs widening.
4558  EVT VT = N->getValueType(0);
4559  assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
4560 
4561  SDValue Cond = GetWidenedVector(N->getOperand(0));
4562  SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4563  SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4564  SDLoc DL(N);
4565 
4566  SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4567  LeftIn, RightIn);
4568  return DAG.getNode(
4569  ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
4570  DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4571 }
4572 
4573 //===----------------------------------------------------------------------===//
4574 // Vector Widening Utilities
4575 //===----------------------------------------------------------------------===//
4576 
4577 // Utility function to find the type to chop up a widen vector for load/store
4578 // TLI: Target lowering used to determine legal types.
4579 // Width: Width left need to load/store.
4580 // WidenVT: The widen vector type to load to/store from
4581 // Align: If 0, don't allow use of a wider type
4582 // WidenEx: If Align is not 0, the amount additional we can load/store from.
4583 
4584 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4585  unsigned Width, EVT WidenVT,
4586  unsigned Align = 0, unsigned WidenEx = 0) {
4587  EVT WidenEltVT = WidenVT.getVectorElementType();
4588  unsigned WidenWidth = WidenVT.getSizeInBits();
4589  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4590  unsigned AlignInBits = Align*8;
4591 
4592  // If we have one element to load/store, return it.
4593  EVT RetVT = WidenEltVT;
4594  if (Width == WidenEltWidth)
4595  return RetVT;
4596 
4597  // See if there is larger legal integer than the element type to load/store.
4598  unsigned VT;
4599  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4600  VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
4601  EVT MemVT((MVT::SimpleValueType) VT);
4602  unsigned MemVTWidth = MemVT.getSizeInBits();
4603  if (MemVT.getSizeInBits() <= WidenEltWidth)
4604  break;
4605  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4606  if ((Action == TargetLowering::TypeLegal ||
4608  (WidenWidth % MemVTWidth) == 0 &&
4609  isPowerOf2_32(WidenWidth / MemVTWidth) &&
4610  (MemVTWidth <= Width ||
4611  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4612  if (MemVTWidth == WidenWidth)
4613  return MemVT;
4614  RetVT = MemVT;
4615  break;
4616  }
4617  }
4618 
4619  // See if there is a larger vector type to load/store that has the same vector
4620  // element type and is evenly divisible with the WidenVT.
4621  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4622  VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
4623  EVT MemVT = (MVT::SimpleValueType) VT;
4624  unsigned MemVTWidth = MemVT.getSizeInBits();
4625  auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4626  if ((Action == TargetLowering::TypeLegal ||
4628  WidenEltVT == MemVT.getVectorElementType() &&
4629  (WidenWidth % MemVTWidth) == 0 &&
4630  isPowerOf2_32(WidenWidth / MemVTWidth) &&
4631  (MemVTWidth <= Width ||
4632  (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4633  if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
4634  return MemVT;
4635  }
4636  }
4637 
4638  return RetVT;
4639 }
4640 
4641 // Builds a vector type from scalar loads
4642 // VecTy: Resulting Vector type
4643 // LDOps: Load operators to build a vector type
4644 // [Start,End) the list of loads to use.
4646  SmallVectorImpl<SDValue> &LdOps,
4647  unsigned Start, unsigned End) {
4648  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4649  SDLoc dl(LdOps[Start]);
4650  EVT LdTy = LdOps[Start].getValueType();
4651  unsigned Width = VecTy.getSizeInBits();
4652  unsigned NumElts = Width / LdTy.getSizeInBits();
4653  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4654 
4655  unsigned Idx = 1;
4656  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4657 
4658  for (unsigned i = Start + 1; i != End; ++i) {
4659  EVT NewLdTy = LdOps[i].getValueType();
4660  if (NewLdTy != LdTy) {
4661  NumElts = Width / NewLdTy.getSizeInBits();
4662  NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4663  VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4664  // Readjust position and vector position based on new load type.
4665  Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4666  LdTy = NewLdTy;
4667  }
4668  VecOp = DAG.getNode(
4669  ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4670  DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4671  }
4672  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4673 }
4674 
4675 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4676  LoadSDNode *LD) {
4677  // The strategy assumes that we can efficiently load power-of-two widths.
4678  // The routine chops the vector into the largest vector loads with the same
4679  // element type or scalar loads and then recombines it to the widen vector
4680  // type.
4681  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4682  unsigned WidenWidth = WidenVT.getSizeInBits();
4683  EVT LdVT = LD->getMemoryVT();
4684  SDLoc dl(LD);
4685  assert(LdVT.isVector() && WidenVT.isVector());
4686  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4687 
4688  // Load information
4689  SDValue Chain = LD->getChain();
4690  SDValue BasePtr = LD->getBasePtr();
4691  unsigned Align = LD->getAlignment();
4692  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4693  AAMDNodes AAInfo = LD->getAAInfo();
4694 
4695  int LdWidth = LdVT.getSizeInBits();
4696  int WidthDiff = WidenWidth - LdWidth;
4697  unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
4698 
4699  // Find the vector type that can load from.
4700  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4701  int NewVTWidth = NewVT.getSizeInBits();
4702  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4703  Align, MMOFlags, AAInfo);
4704  LdChain.push_back(LdOp.getValue(1));
4705 
4706  // Check if we can load the element with one instruction.
4707  if (LdWidth <= NewVTWidth) {
4708  if (!NewVT.isVector()) {
4709  unsigned NumElts = WidenWidth / NewVTWidth;
4710  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4711  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4712  return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4713  }
4714  if (NewVT == WidenVT)
4715  return LdOp;
4716 
4717  assert(WidenWidth % NewVTWidth == 0);
4718  unsigned NumConcat = WidenWidth / NewVTWidth;
4719  SmallVector<SDValue, 16> ConcatOps(NumConcat);
4720  SDValue UndefVal = DAG.getUNDEF(NewVT);
4721  ConcatOps[0] = LdOp;
4722  for (unsigned i = 1; i != NumConcat; ++i)
4723  ConcatOps[i] = UndefVal;
4724  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4725  }
4726 
4727  // Load vector by using multiple loads from largest vector to scalar.
4729  LdOps.push_back(LdOp);
4730 
4731  LdWidth -= NewVTWidth;
4732  unsigned Offset = 0;
4733 
4734  while (LdWidth > 0) {
4735  unsigned Increment = NewVTWidth / 8;
4736  Offset += Increment;
4737  BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4738 
4739  SDValue L;
4740  if (LdWidth < NewVTWidth) {
4741  // The current type we are using is too large. Find a better size.
4742  NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4743  NewVTWidth = NewVT.getSizeInBits();
4744  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4745  LD->getPointerInfo().getWithOffset(Offset),
4746  MinAlign(Align, Increment), MMOFlags, AAInfo);
4747  LdChain.push_back(L.getValue(1));
4748  if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
4749  // Later code assumes the vector loads produced will be mergeable, so we
4750  // must pad the final entry up to the previous width. Scalars are
4751  // combined separately.
4753  Loads.push_back(L);
4754  unsigned size = L->getValueSizeInBits(0);
4755  while (size < LdOp->getValueSizeInBits(0)) {
4756  Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4757  size += L->getValueSizeInBits(0);
4758  }
4759  L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4760  }
4761  } else {
4762  L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4763  LD->getPointerInfo().getWithOffset(Offset),
4764  MinAlign(Align, Increment), MMOFlags, AAInfo);
4765  LdChain.push_back(L.getValue(1));
4766  }
4767 
4768  LdOps.push_back(L);
4769  LdOp = L;
4770 
4771  LdWidth -= NewVTWidth;
4772  }
4773 
4774  // Build the vector from the load operations.
4775  unsigned End = LdOps.size();
4776  if (!LdOps[0].getValueType().isVector())
4777  // All the loads are scalar loads.
4778  return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4779 
4780  // If the load contains vectors, build the vector using concat vector.
4781  // All of the vectors used to load are power-of-2, and the scalar loads can be
4782  // combined to make a power-of-2 vector.
4783  SmallVector<SDValue, 16> ConcatOps(End);
4784  int i = End - 1;
4785  int Idx = End;
4786  EVT LdTy = LdOps[i].getValueType();
4787  // First, combine the scalar loads to a vector.
4788  if (!LdTy.isVector()) {
4789  for (--i; i >= 0; --i) {
4790  LdTy = LdOps[i].getValueType();
4791  if (LdTy.isVector())
4792  break;
4793  }
4794  ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4795  }
4796  ConcatOps[--Idx] = LdOps[i];
4797  for (--i; i >= 0; --i) {
4798  EVT NewLdTy = LdOps[i].getValueType();
4799  if (NewLdTy != LdTy) {
4800  // Create a larger vector.
4801  ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4802  makeArrayRef(&ConcatOps[Idx], End - Idx));
4803  Idx = End - 1;
4804  LdTy = NewLdTy;
4805  }
4806  ConcatOps[--Idx] = LdOps[i];
4807  }
4808 
4809  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
4810  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4811  makeArrayRef(&ConcatOps[Idx], End - Idx));
4812 
4813  // We need to fill the rest with undefs to build the vector.
4814  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
4815  SmallVector<SDValue, 16> WidenOps(NumOps);
4816  SDValue UndefVal = DAG.getUNDEF(LdTy);
4817  {
4818  unsigned i = 0;
4819  for (; i != End-Idx; ++i)
4820  WidenOps[i] = ConcatOps[Idx+i];
4821  for (; i != NumOps; ++i)
4822  WidenOps[i] = UndefVal;
4823  }
4824  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
4825 }
4826 
4827 SDValue
4828 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
4829  LoadSDNode *LD,
4831  // For extension loads, it may not be more efficient to chop up the vector
4832  // and then extend it. Instead, we unroll the load and build a new vector.
4833  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4834  EVT LdVT = LD->getMemoryVT();
4835  SDLoc dl(LD);
4836  assert(LdVT.isVector() && WidenVT.isVector());
4837 
4838  // Load information
4839  SDValue Chain = LD->getChain();
4840  SDValue BasePtr = LD->getBasePtr();
4841  unsigned Align = LD->getAlignment();
4842  MachineMemOperand::Flags MMOFlags = LD->getMemOpe