LLVM  14.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
26 #include "llvm/Support/TypeSize.h"
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "legalize-types"
31 
32 //===----------------------------------------------------------------------===//
33 // Result Vector Scalarization: <1 x ty> -> ty.
34 //===----------------------------------------------------------------------===//
35 
36 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
37  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
38  dbgs() << "\n");
39  SDValue R = SDValue();
40 
41  switch (N->getOpcode()) {
42  default:
43 #ifndef NDEBUG
44  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45  N->dump(&DAG);
46  dbgs() << "\n";
47 #endif
48  report_fatal_error("Do not know how to scalarize the result of this "
49  "operator!\n");
50 
51  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
70  R = ScalarizeVecRes_VecInregOp(N);
71  break;
72  case ISD::ABS:
73  case ISD::ANY_EXTEND:
74  case ISD::BITREVERSE:
75  case ISD::BSWAP:
76  case ISD::CTLZ:
78  case ISD::CTPOP:
79  case ISD::CTTZ:
81  case ISD::FABS:
82  case ISD::FCEIL:
83  case ISD::FCOS:
84  case ISD::FEXP:
85  case ISD::FEXP2:
86  case ISD::FFLOOR:
87  case ISD::FLOG:
88  case ISD::FLOG10:
89  case ISD::FLOG2:
90  case ISD::FNEARBYINT:
91  case ISD::FNEG:
92  case ISD::FREEZE:
93  case ISD::ARITH_FENCE:
94  case ISD::FP_EXTEND:
95  case ISD::FP_TO_SINT:
96  case ISD::FP_TO_UINT:
97  case ISD::FRINT:
98  case ISD::FROUND:
99  case ISD::FROUNDEVEN:
100  case ISD::FSIN:
101  case ISD::FSQRT:
102  case ISD::FTRUNC:
103  case ISD::SIGN_EXTEND:
104  case ISD::SINT_TO_FP:
105  case ISD::TRUNCATE:
106  case ISD::UINT_TO_FP:
107  case ISD::ZERO_EXTEND:
108  case ISD::FCANONICALIZE:
109  R = ScalarizeVecRes_UnaryOp(N);
110  break;
111 
112  case ISD::ADD:
113  case ISD::AND:
114  case ISD::FADD:
115  case ISD::FCOPYSIGN:
116  case ISD::FDIV:
117  case ISD::FMUL:
118  case ISD::FMINNUM:
119  case ISD::FMAXNUM:
120  case ISD::FMINNUM_IEEE:
121  case ISD::FMAXNUM_IEEE:
122  case ISD::FMINIMUM:
123  case ISD::FMAXIMUM:
124  case ISD::SMIN:
125  case ISD::SMAX:
126  case ISD::UMIN:
127  case ISD::UMAX:
128 
129  case ISD::SADDSAT:
130  case ISD::UADDSAT:
131  case ISD::SSUBSAT:
132  case ISD::USUBSAT:
133  case ISD::SSHLSAT:
134  case ISD::USHLSAT:
135 
136  case ISD::FPOW:
137  case ISD::FREM:
138  case ISD::FSUB:
139  case ISD::MUL:
140  case ISD::OR:
141  case ISD::SDIV:
142  case ISD::SREM:
143  case ISD::SUB:
144  case ISD::UDIV:
145  case ISD::UREM:
146  case ISD::XOR:
147  case ISD::SHL:
148  case ISD::SRA:
149  case ISD::SRL:
150  case ISD::ROTL:
151  case ISD::ROTR:
152  R = ScalarizeVecRes_BinOp(N);
153  break;
154  case ISD::FMA:
155  case ISD::FSHL:
156  case ISD::FSHR:
157  R = ScalarizeVecRes_TernaryOp(N);
158  break;
159 
160 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
161  case ISD::STRICT_##DAGN:
162 #include "llvm/IR/ConstrainedOps.def"
163  R = ScalarizeVecRes_StrictFPOp(N);
164  break;
165 
166  case ISD::FP_TO_UINT_SAT:
167  case ISD::FP_TO_SINT_SAT:
168  R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
169  break;
170 
171  case ISD::UADDO:
172  case ISD::SADDO:
173  case ISD::USUBO:
174  case ISD::SSUBO:
175  case ISD::UMULO:
176  case ISD::SMULO:
177  R = ScalarizeVecRes_OverflowOp(N, ResNo);
178  break;
179  case ISD::SMULFIX:
180  case ISD::SMULFIXSAT:
181  case ISD::UMULFIX:
182  case ISD::UMULFIXSAT:
183  case ISD::SDIVFIX:
184  case ISD::SDIVFIXSAT:
185  case ISD::UDIVFIX:
186  case ISD::UDIVFIXSAT:
187  R = ScalarizeVecRes_FIX(N);
188  break;
189  }
190 
191  // If R is null, the sub-method took care of registering the result.
192  if (R.getNode())
193  SetScalarizedVector(SDValue(N, ResNo), R);
194 }
195 
196 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
197  SDValue LHS = GetScalarizedVector(N->getOperand(0));
198  SDValue RHS = GetScalarizedVector(N->getOperand(1));
199  return DAG.getNode(N->getOpcode(), SDLoc(N),
200  LHS.getValueType(), LHS, RHS, N->getFlags());
201 }
202 
203 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
204  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
205  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
206  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
207  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
208  Op2, N->getFlags());
209 }
210 
211 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
212  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
213  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
214  SDValue Op2 = N->getOperand(2);
215  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
216  Op2, N->getFlags());
217 }
218 
219 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
220  EVT VT = N->getValueType(0).getVectorElementType();
221  unsigned NumOpers = N->getNumOperands();
222  SDValue Chain = N->getOperand(0);
223  EVT ValueVTs[] = {VT, MVT::Other};
224  SDLoc dl(N);
225 
226  SmallVector<SDValue, 4> Opers(NumOpers);
227 
228  // The Chain is the first operand.
229  Opers[0] = Chain;
230 
231  // Now process the remaining operands.
232  for (unsigned i = 1; i < NumOpers; ++i) {
233  SDValue Oper = N->getOperand(i);
234 
235  if (Oper.getValueType().isVector())
236  Oper = GetScalarizedVector(Oper);
237 
238  Opers[i] = Oper;
239  }
240 
241  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
242  Opers, N->getFlags());
243 
244  // Legalize the chain result - switch anything that used the old chain to
245  // use the new one.
246  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
247  return Result;
248 }
249 
250 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
251  unsigned ResNo) {
252  SDLoc DL(N);
253  EVT ResVT = N->getValueType(0);
254  EVT OvVT = N->getValueType(1);
255 
256  SDValue ScalarLHS, ScalarRHS;
257  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
258  ScalarLHS = GetScalarizedVector(N->getOperand(0));
259  ScalarRHS = GetScalarizedVector(N->getOperand(1));
260  } else {
261  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
262  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
263  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
264  ScalarLHS = ElemsLHS[0];
265  ScalarRHS = ElemsRHS[0];
266  }
267 
268  SDVTList ScalarVTs = DAG.getVTList(
270  SDNode *ScalarNode = DAG.getNode(
271  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
272  ScalarNode->setFlags(N->getFlags());
273 
274  // Replace the other vector result not being explicitly scalarized here.
275  unsigned OtherNo = 1 - ResNo;
276  EVT OtherVT = N->getValueType(OtherNo);
277  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
278  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
279  } else {
280  SDValue OtherVal = DAG.getNode(
281  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
282  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
283  }
284 
285  return SDValue(ScalarNode, ResNo);
286 }
287 
288 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
289  unsigned ResNo) {
290  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
291  return GetScalarizedVector(Op);
292 }
293 
294 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
295  SDValue Op = N->getOperand(0);
296  if (Op.getValueType().isVector()
297  && Op.getValueType().getVectorNumElements() == 1
298  && !isSimpleLegalType(Op.getValueType()))
299  Op = GetScalarizedVector(Op);
300  EVT NewVT = N->getValueType(0).getVectorElementType();
301  return DAG.getNode(ISD::BITCAST, SDLoc(N),
302  NewVT, Op);
303 }
304 
305 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
306  EVT EltVT = N->getValueType(0).getVectorElementType();
307  SDValue InOp = N->getOperand(0);
308  // The BUILD_VECTOR operands may be of wider element types and
309  // we may need to truncate them back to the requested return type.
310  if (EltVT.isInteger())
311  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
312  return InOp;
313 }
314 
315 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
316  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
317  N->getValueType(0).getVectorElementType(),
318  N->getOperand(0), N->getOperand(1));
319 }
320 
321 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
322  SDLoc DL(N);
323  SDValue Op = N->getOperand(0);
324  EVT OpVT = Op.getValueType();
325  // The result needs scalarizing, but it's not a given that the source does.
326  // See similar logic in ScalarizeVecRes_UnaryOp.
327  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
328  Op = GetScalarizedVector(Op);
329  } else {
330  EVT VT = OpVT.getVectorElementType();
331  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
332  DAG.getVectorIdxConstant(0, DL));
333  }
334  return DAG.getNode(ISD::FP_ROUND, DL,
335  N->getValueType(0).getVectorElementType(), Op,
336  N->getOperand(1));
337 }
338 
339 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
340  SDValue Op = GetScalarizedVector(N->getOperand(0));
341  return DAG.getNode(ISD::FPOWI, SDLoc(N),
342  Op.getValueType(), Op, N->getOperand(1));
343 }
344 
345 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
346  // The value to insert may have a wider type than the vector element type,
347  // so be sure to truncate it to the element type if necessary.
348  SDValue Op = N->getOperand(1);
349  EVT EltVT = N->getValueType(0).getVectorElementType();
350  if (Op.getValueType() != EltVT)
351  // FIXME: Can this happen for floating point types?
352  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
353  return Op;
354 }
355 
356 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
357  assert(N->isUnindexed() && "Indexed vector load?");
358 
359  SDValue Result = DAG.getLoad(
360  ISD::UNINDEXED, N->getExtensionType(),
361  N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
362  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
363  N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
364  N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
365 
366  // Legalize the chain result - switch anything that used the old chain to
367  // use the new one.
368  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
369  return Result;
370 }
371 
372 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
373  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
374  EVT DestVT = N->getValueType(0).getVectorElementType();
375  SDValue Op = N->getOperand(0);
376  EVT OpVT = Op.getValueType();
377  SDLoc DL(N);
378  // The result needs scalarizing, but it's not a given that the source does.
379  // This is a workaround for targets where it's impossible to scalarize the
380  // result of a conversion, because the source type is legal.
381  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
382  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
383  // legal and was not scalarized.
384  // See the similar logic in ScalarizeVecRes_SETCC
385  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
386  Op = GetScalarizedVector(Op);
387  } else {
388  EVT VT = OpVT.getVectorElementType();
389  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
390  DAG.getVectorIdxConstant(0, DL));
391  }
392  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
393 }
394 
395 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
396  EVT EltVT = N->getValueType(0).getVectorElementType();
397  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
398  SDValue LHS = GetScalarizedVector(N->getOperand(0));
399  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
400  LHS, DAG.getValueType(ExtVT));
401 }
402 
403 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
404  SDLoc DL(N);
405  SDValue Op = N->getOperand(0);
406 
407  EVT OpVT = Op.getValueType();
408  EVT OpEltVT = OpVT.getVectorElementType();
409  EVT EltVT = N->getValueType(0).getVectorElementType();
410 
411  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
412  Op = GetScalarizedVector(Op);
413  } else {
414  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
415  DAG.getVectorIdxConstant(0, DL));
416  }
417 
418  switch (N->getOpcode()) {
420  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
422  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
424  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
425  }
426 
427  llvm_unreachable("Illegal extend_vector_inreg opcode");
428 }
429 
430 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
431  // If the operand is wider than the vector element type then it is implicitly
432  // truncated. Make that explicit here.
433  EVT EltVT = N->getValueType(0).getVectorElementType();
434  SDValue InOp = N->getOperand(0);
435  if (InOp.getValueType() != EltVT)
436  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
437  return InOp;
438 }
439 
440 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
441  SDValue Cond = N->getOperand(0);
442  EVT OpVT = Cond.getValueType();
443  SDLoc DL(N);
444  // The vselect result and true/value operands needs scalarizing, but it's
445  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
446  // See the similar logic in ScalarizeVecRes_SETCC
447  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
448  Cond = GetScalarizedVector(Cond);
449  } else {
450  EVT VT = OpVT.getVectorElementType();
452  DAG.getVectorIdxConstant(0, DL));
453  }
454 
455  SDValue LHS = GetScalarizedVector(N->getOperand(1));
456  TargetLowering::BooleanContent ScalarBool =
457  TLI.getBooleanContents(false, false);
458  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
459 
460  // If integer and float booleans have different contents then we can't
461  // reliably optimize in all cases. There is a full explanation for this in
462  // DAGCombiner::visitSELECT() where the same issue affects folding
463  // (select C, 0, 1) to (xor C, 1).
464  if (TLI.getBooleanContents(false, false) !=
465  TLI.getBooleanContents(false, true)) {
466  // At least try the common case where the boolean is generated by a
467  // comparison.
468  if (Cond->getOpcode() == ISD::SETCC) {
469  EVT OpVT = Cond->getOperand(0).getValueType();
470  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
471  VecBool = TLI.getBooleanContents(OpVT);
472  } else
474  }
475 
476  EVT CondVT = Cond.getValueType();
477  if (ScalarBool != VecBool) {
478  switch (ScalarBool) {
480  break;
484  // Vector read from all ones, scalar expects a single 1 so mask.
485  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
486  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
487  break;
491  // Vector reads from a one, scalar from all ones so sign extend.
492  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
493  Cond, DAG.getValueType(MVT::i1));
494  break;
495  }
496  }
497 
498  // Truncate the condition if needed
499  auto BoolVT = getSetCCResultType(CondVT);
500  if (BoolVT.bitsLT(CondVT))
501  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
502 
503  return DAG.getSelect(SDLoc(N),
504  LHS.getValueType(), Cond, LHS,
505  GetScalarizedVector(N->getOperand(2)));
506 }
507 
508 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
509  SDValue LHS = GetScalarizedVector(N->getOperand(1));
510  return DAG.getSelect(SDLoc(N),
511  LHS.getValueType(), N->getOperand(0), LHS,
512  GetScalarizedVector(N->getOperand(2)));
513 }
514 
515 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
516  SDValue LHS = GetScalarizedVector(N->getOperand(2));
517  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
518  N->getOperand(0), N->getOperand(1),
519  LHS, GetScalarizedVector(N->getOperand(3)),
520  N->getOperand(4));
521 }
522 
523 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
524  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
525 }
526 
527 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
528  // Figure out if the scalar is the LHS or RHS and return it.
529  SDValue Arg = N->getOperand(2).getOperand(0);
530  if (Arg.isUndef())
531  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
532  unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
533  return GetScalarizedVector(N->getOperand(Op));
534 }
535 
536 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
537  SDValue Src = N->getOperand(0);
538  EVT SrcVT = Src.getValueType();
539  SDLoc dl(N);
540 
541  // Handle case where result is scalarized but operand is not
542  if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
543  Src = GetScalarizedVector(Src);
544  else
545  Src = DAG.getNode(
547  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
548 
549  EVT DstVT = N->getValueType(0).getVectorElementType();
550  return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
551 }
552 
553 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
554  assert(N->getValueType(0).isVector() &&
555  N->getOperand(0).getValueType().isVector() &&
556  "Operand types must be vectors");
557  SDValue LHS = N->getOperand(0);
558  SDValue RHS = N->getOperand(1);
559  EVT OpVT = LHS.getValueType();
560  EVT NVT = N->getValueType(0).getVectorElementType();
561  SDLoc DL(N);
562 
563  // The result needs scalarizing, but it's not a given that the source does.
564  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
565  LHS = GetScalarizedVector(LHS);
566  RHS = GetScalarizedVector(RHS);
567  } else {
568  EVT VT = OpVT.getVectorElementType();
569  LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
570  DAG.getVectorIdxConstant(0, DL));
571  RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
572  DAG.getVectorIdxConstant(0, DL));
573  }
574 
575  // Turn it into a scalar SETCC.
576  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
577  N->getOperand(2));
578  // Vectors may have a different boolean contents to scalars. Promote the
579  // value appropriately.
580  ISD::NodeType ExtendCode =
582  return DAG.getNode(ExtendCode, DL, NVT, Res);
583 }
584 
585 
586 //===----------------------------------------------------------------------===//
587 // Operand Vector Scalarization <1 x ty> -> ty.
588 //===----------------------------------------------------------------------===//
589 
590 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
591  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
592  dbgs() << "\n");
593  SDValue Res = SDValue();
594 
595  switch (N->getOpcode()) {
596  default:
597 #ifndef NDEBUG
598  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
599  N->dump(&DAG);
600  dbgs() << "\n";
601 #endif
602  report_fatal_error("Do not know how to scalarize this operator's "
603  "operand!\n");
604  case ISD::BITCAST:
605  Res = ScalarizeVecOp_BITCAST(N);
606  break;
607  case ISD::ANY_EXTEND:
608  case ISD::ZERO_EXTEND:
609  case ISD::SIGN_EXTEND:
610  case ISD::TRUNCATE:
611  case ISD::FP_TO_SINT:
612  case ISD::FP_TO_UINT:
613  case ISD::SINT_TO_FP:
614  case ISD::UINT_TO_FP:
615  Res = ScalarizeVecOp_UnaryOp(N);
616  break;
621  Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
622  break;
623  case ISD::CONCAT_VECTORS:
624  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
625  break;
627  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
628  break;
629  case ISD::VSELECT:
630  Res = ScalarizeVecOp_VSELECT(N);
631  break;
632  case ISD::SETCC:
633  Res = ScalarizeVecOp_VSETCC(N);
634  break;
635  case ISD::STORE:
636  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
637  break;
639  Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
640  break;
641  case ISD::FP_ROUND:
642  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
643  break;
645  Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
646  break;
647  case ISD::FP_EXTEND:
648  Res = ScalarizeVecOp_FP_EXTEND(N);
649  break;
650  case ISD::VECREDUCE_FADD:
651  case ISD::VECREDUCE_FMUL:
652  case ISD::VECREDUCE_ADD:
653  case ISD::VECREDUCE_MUL:
654  case ISD::VECREDUCE_AND:
655  case ISD::VECREDUCE_OR:
656  case ISD::VECREDUCE_XOR:
657  case ISD::VECREDUCE_SMAX:
658  case ISD::VECREDUCE_SMIN:
659  case ISD::VECREDUCE_UMAX:
660  case ISD::VECREDUCE_UMIN:
661  case ISD::VECREDUCE_FMAX:
662  case ISD::VECREDUCE_FMIN:
663  Res = ScalarizeVecOp_VECREDUCE(N);
664  break;
667  Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
668  break;
669  }
670 
671  // If the result is null, the sub-method took care of registering results etc.
672  if (!Res.getNode()) return false;
673 
674  // If the result is N, the sub-method updated N in place. Tell the legalizer
675  // core about this.
676  if (Res.getNode() == N)
677  return true;
678 
679  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
680  "Invalid operand expansion");
681 
682  ReplaceValueWith(SDValue(N, 0), Res);
683  return false;
684 }
685 
686 /// If the value to convert is a vector that needs to be scalarized, it must be
687 /// <1 x ty>. Convert the element instead.
688 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
689  SDValue Elt = GetScalarizedVector(N->getOperand(0));
690  return DAG.getNode(ISD::BITCAST, SDLoc(N),
691  N->getValueType(0), Elt);
692 }
693 
694 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
695 /// Do the operation on the element instead.
696 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
697  assert(N->getValueType(0).getVectorNumElements() == 1 &&
698  "Unexpected vector type!");
699  SDValue Elt = GetScalarizedVector(N->getOperand(0));
700  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
701  N->getValueType(0).getScalarType(), Elt);
702  // Revectorize the result so the types line up with what the uses of this
703  // expression expect.
704  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
705 }
706 
707 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
708 /// Do the strict FP operation on the element instead.
709 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
710  assert(N->getValueType(0).getVectorNumElements() == 1 &&
711  "Unexpected vector type!");
712  SDValue Elt = GetScalarizedVector(N->getOperand(1));
713  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
714  { N->getValueType(0).getScalarType(), MVT::Other },
715  { N->getOperand(0), Elt });
716  // Legalize the chain result - switch anything that used the old chain to
717  // use the new one.
718  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
719  // Revectorize the result so the types line up with what the uses of this
720  // expression expect.
721  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
722 
723  // Do our own replacement and return SDValue() to tell the caller that we
724  // handled all replacements since caller can only handle a single result.
725  ReplaceValueWith(SDValue(N, 0), Res);
726  return SDValue();
727 }
728 
729 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
730 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
731  SmallVector<SDValue, 8> Ops(N->getNumOperands());
732  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
733  Ops[i] = GetScalarizedVector(N->getOperand(i));
734  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
735 }
736 
737 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
738 /// so just return the element, ignoring the index.
739 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
740  EVT VT = N->getValueType(0);
741  SDValue Res = GetScalarizedVector(N->getOperand(0));
742  if (Res.getValueType() != VT)
743  Res = VT.isFloatingPoint()
744  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
745  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
746  return Res;
747 }
748 
749 /// If the input condition is a vector that needs to be scalarized, it must be
750 /// <1 x i1>, so just convert to a normal ISD::SELECT
751 /// (still with vector output type since that was acceptable if we got here).
752 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
753  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
754  EVT VT = N->getValueType(0);
755 
756  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
757  N->getOperand(2));
758 }
759 
760 /// If the operand is a vector that needs to be scalarized then the
761 /// result must be v1i1, so just convert to a scalar SETCC and wrap
762 /// with a scalar_to_vector since the res type is legal if we got here
763 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
764  assert(N->getValueType(0).isVector() &&
765  N->getOperand(0).getValueType().isVector() &&
766  "Operand types must be vectors");
767  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
768 
769  EVT VT = N->getValueType(0);
770  SDValue LHS = GetScalarizedVector(N->getOperand(0));
771  SDValue RHS = GetScalarizedVector(N->getOperand(1));
772 
773  EVT OpVT = N->getOperand(0).getValueType();
774  EVT NVT = VT.getVectorElementType();
775  SDLoc DL(N);
776  // Turn it into a scalar SETCC.
777  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
778  N->getOperand(2));
779 
780  // Vectors may have a different boolean contents to scalars. Promote the
781  // value appropriately.
782  ISD::NodeType ExtendCode =
784 
785  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
786 
787  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
788 }
789 
790 /// If the value to store is a vector that needs to be scalarized, it must be
791 /// <1 x ty>. Just store the element.
792 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
793  assert(N->isUnindexed() && "Indexed store of one-element vector?");
794  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
795  SDLoc dl(N);
796 
797  if (N->isTruncatingStore())
798  return DAG.getTruncStore(
799  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
800  N->getBasePtr(), N->getPointerInfo(),
801  N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
802  N->getMemOperand()->getFlags(), N->getAAInfo());
803 
804  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
805  N->getBasePtr(), N->getPointerInfo(),
806  N->getOriginalAlign(), N->getMemOperand()->getFlags(),
807  N->getAAInfo());
808 }
809 
810 /// If the value to round is a vector that needs to be scalarized, it must be
811 /// <1 x ty>. Convert the element instead.
812 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
813  assert(OpNo == 0 && "Wrong operand for scalarization!");
814  SDValue Elt = GetScalarizedVector(N->getOperand(0));
815  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
816  N->getValueType(0).getVectorElementType(), Elt,
817  N->getOperand(1));
818  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
819 }
820 
821 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
822  unsigned OpNo) {
823  assert(OpNo == 1 && "Wrong operand for scalarization!");
824  SDValue Elt = GetScalarizedVector(N->getOperand(1));
826  { N->getValueType(0).getVectorElementType(),
827  MVT::Other },
828  { N->getOperand(0), Elt, N->getOperand(2) });
829  // Legalize the chain result - switch anything that used the old chain to
830  // use the new one.
831  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
832 
833  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
834 
835  // Do our own replacement and return SDValue() to tell the caller that we
836  // handled all replacements since caller can only handle a single result.
837  ReplaceValueWith(SDValue(N, 0), Res);
838  return SDValue();
839 }
840 
841 /// If the value to extend is a vector that needs to be scalarized, it must be
842 /// <1 x ty>. Convert the element instead.
843 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
844  SDValue Elt = GetScalarizedVector(N->getOperand(0));
845  SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
846  N->getValueType(0).getVectorElementType(), Elt);
847  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
848 }
849 
850 /// If the value to extend is a vector that needs to be scalarized, it must be
851 /// <1 x ty>. Convert the element instead.
852 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
853  SDValue Elt = GetScalarizedVector(N->getOperand(1));
854  SDValue Res =
856  {N->getValueType(0).getVectorElementType(), MVT::Other},
857  {N->getOperand(0), Elt});
858  // Legalize the chain result - switch anything that used the old chain to
859  // use the new one.
860  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
861 
862  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
863 
864  // Do our own replacement and return SDValue() to tell the caller that we
865  // handled all replacements since caller can only handle a single result.
866  ReplaceValueWith(SDValue(N, 0), Res);
867  return SDValue();
868 }
869 
870 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
871  SDValue Res = GetScalarizedVector(N->getOperand(0));
872  // Result type may be wider than element type.
873  if (Res.getValueType() != N->getValueType(0))
874  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
875  return Res;
876 }
877 
878 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
879  SDValue AccOp = N->getOperand(0);
880  SDValue VecOp = N->getOperand(1);
881 
882  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
883 
884  SDValue Op = GetScalarizedVector(VecOp);
885  return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
886  AccOp, Op, N->getFlags());
887 }
888 
889 //===----------------------------------------------------------------------===//
890 // Result Vector Splitting
891 //===----------------------------------------------------------------------===//
892 
893 /// This method is called when the specified result of the specified node is
894 /// found to need vector splitting. At this point, the node may also have
895 /// invalid operands or may have other results that need legalization, we just
896 /// know that (at least) one result needs vector splitting.
897 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
898  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
899  SDValue Lo, Hi;
900 
901  // See if the target wants to custom expand this node.
902  if (CustomLowerNode(N, N->getValueType(ResNo), true))
903  return;
904 
905  switch (N->getOpcode()) {
906  default:
907 #ifndef NDEBUG
908  dbgs() << "SplitVectorResult #" << ResNo << ": ";
909  N->dump(&DAG);
910  dbgs() << "\n";
911 #endif
912  report_fatal_error("Do not know how to split the result of this "
913  "operator!\n");
914 
915  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
916  case ISD::VSELECT:
917  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
918  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
919  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
920  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
921  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
922  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
923  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
924  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
925  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
926  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
927  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
928  case ISD::SPLAT_VECTOR:
930  SplitVecRes_ScalarOp(N, Lo, Hi);
931  break;
932  case ISD::STEP_VECTOR:
933  SplitVecRes_STEP_VECTOR(N, Lo, Hi);
934  break;
935  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
936  case ISD::LOAD:
937  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
938  break;
939  case ISD::MLOAD:
940  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
941  break;
942  case ISD::MGATHER:
943  SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
944  break;
945  case ISD::SETCC:
946  SplitVecRes_SETCC(N, Lo, Hi);
947  break;
948  case ISD::VECTOR_REVERSE:
949  SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
950  break;
951  case ISD::VECTOR_SHUFFLE:
952  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
953  break;
954  case ISD::VECTOR_SPLICE:
955  SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
956  break;
957  case ISD::VAARG:
958  SplitVecRes_VAARG(N, Lo, Hi);
959  break;
960 
964  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
965  break;
966 
967  case ISD::ABS:
968  case ISD::BITREVERSE:
969  case ISD::BSWAP:
970  case ISD::CTLZ:
971  case ISD::CTTZ:
974  case ISD::CTPOP:
975  case ISD::FABS:
976  case ISD::FCEIL:
977  case ISD::FCOS:
978  case ISD::FEXP:
979  case ISD::FEXP2:
980  case ISD::FFLOOR:
981  case ISD::FLOG:
982  case ISD::FLOG10:
983  case ISD::FLOG2:
984  case ISD::FNEARBYINT:
985  case ISD::FNEG:
986  case ISD::FREEZE:
987  case ISD::ARITH_FENCE:
988  case ISD::FP_EXTEND:
989  case ISD::FP_ROUND:
990  case ISD::FP_TO_SINT:
991  case ISD::FP_TO_UINT:
992  case ISD::FRINT:
993  case ISD::FROUND:
994  case ISD::FROUNDEVEN:
995  case ISD::FSIN:
996  case ISD::FSQRT:
997  case ISD::FTRUNC:
998  case ISD::SINT_TO_FP:
999  case ISD::TRUNCATE:
1000  case ISD::UINT_TO_FP:
1001  case ISD::FCANONICALIZE:
1002  SplitVecRes_UnaryOp(N, Lo, Hi);
1003  break;
1004 
1005  case ISD::ANY_EXTEND:
1006  case ISD::SIGN_EXTEND:
1007  case ISD::ZERO_EXTEND:
1008  SplitVecRes_ExtendOp(N, Lo, Hi);
1009  break;
1010 
1011  case ISD::ADD:
1012  case ISD::SUB:
1013  case ISD::MUL:
1014  case ISD::MULHS:
1015  case ISD::MULHU:
1016  case ISD::FADD:
1017  case ISD::FSUB:
1018  case ISD::FMUL:
1019  case ISD::FMINNUM:
1020  case ISD::FMAXNUM:
1021  case ISD::FMINIMUM:
1022  case ISD::FMAXIMUM:
1023  case ISD::SDIV:
1024  case ISD::UDIV:
1025  case ISD::FDIV:
1026  case ISD::FPOW:
1027  case ISD::AND:
1028  case ISD::OR:
1029  case ISD::XOR:
1030  case ISD::SHL:
1031  case ISD::SRA:
1032  case ISD::SRL:
1033  case ISD::UREM:
1034  case ISD::SREM:
1035  case ISD::FREM:
1036  case ISD::SMIN:
1037  case ISD::SMAX:
1038  case ISD::UMIN:
1039  case ISD::UMAX:
1040  case ISD::SADDSAT:
1041  case ISD::UADDSAT:
1042  case ISD::SSUBSAT:
1043  case ISD::USUBSAT:
1044  case ISD::SSHLSAT:
1045  case ISD::USHLSAT:
1046  case ISD::ROTL:
1047  case ISD::ROTR:
1048  SplitVecRes_BinOp(N, Lo, Hi, /*IsVP*/ false);
1049  break;
1050  case ISD::FMA:
1051  case ISD::FSHL:
1052  case ISD::FSHR:
1053  SplitVecRes_TernaryOp(N, Lo, Hi);
1054  break;
1055 
1056 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1057  case ISD::STRICT_##DAGN:
1058 #include "llvm/IR/ConstrainedOps.def"
1059  SplitVecRes_StrictFPOp(N, Lo, Hi);
1060  break;
1061 
1062  case ISD::FP_TO_UINT_SAT:
1063  case ISD::FP_TO_SINT_SAT:
1064  SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1065  break;
1066 
1067  case ISD::UADDO:
1068  case ISD::SADDO:
1069  case ISD::USUBO:
1070  case ISD::SSUBO:
1071  case ISD::UMULO:
1072  case ISD::SMULO:
1073  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1074  break;
1075  case ISD::SMULFIX:
1076  case ISD::SMULFIXSAT:
1077  case ISD::UMULFIX:
1078  case ISD::UMULFIXSAT:
1079  case ISD::SDIVFIX:
1080  case ISD::SDIVFIXSAT:
1081  case ISD::UDIVFIX:
1082  case ISD::UDIVFIXSAT:
1083  SplitVecRes_FIX(N, Lo, Hi);
1084  break;
1085  case ISD::VP_ADD:
1086  case ISD::VP_AND:
1087  case ISD::VP_MUL:
1088  case ISD::VP_OR:
1089  case ISD::VP_SUB:
1090  case ISD::VP_XOR:
1091  case ISD::VP_SHL:
1092  case ISD::VP_LSHR:
1093  case ISD::VP_ASHR:
1094  case ISD::VP_SDIV:
1095  case ISD::VP_UDIV:
1096  case ISD::VP_SREM:
1097  case ISD::VP_UREM:
1098  case ISD::VP_FADD:
1099  case ISD::VP_FSUB:
1100  case ISD::VP_FMUL:
1101  case ISD::VP_FDIV:
1102  case ISD::VP_FREM:
1103  SplitVecRes_BinOp(N, Lo, Hi, /*IsVP*/ true);
1104  break;
1105  }
1106 
1107  // If Lo/Hi is null, the sub-method took care of registering results etc.
1108  if (Lo.getNode())
1109  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1110 }
1111 
1112 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1113  MachinePointerInfo &MPI, SDValue &Ptr,
1114  uint64_t *ScaledOffset) {
1115  SDLoc DL(N);
1116  unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1117 
1118  if (MemVT.isScalableVector()) {
1119  SDNodeFlags Flags;
1120  SDValue BytesIncrement = DAG.getVScale(
1121  DL, Ptr.getValueType(),
1122  APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1123  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1124  Flags.setNoUnsignedWrap(true);
1125  if (ScaledOffset)
1126  *ScaledOffset += IncrementSize;
1127  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1128  Flags);
1129  } else {
1130  MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1131  // Increment the pointer to the other half.
1132  Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1133  }
1134 }
1135 
1136 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi,
1137  bool IsVP) {
1138  SDValue LHSLo, LHSHi;
1139  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1140  SDValue RHSLo, RHSHi;
1141  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1142  SDLoc dl(N);
1143 
1144  const SDNodeFlags Flags = N->getFlags();
1145  unsigned Opcode = N->getOpcode();
1146  if (!IsVP) {
1147  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1148  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1149  return;
1150  }
1151 
1152  // Split the mask.
1153  SDValue MaskLo, MaskHi;
1154  SDValue Mask = N->getOperand(2);
1155  EVT MaskVT = Mask.getValueType();
1156  if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1157  GetSplitVector(Mask, MaskLo, MaskHi);
1158  else
1159  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, SDLoc(Mask));
1160 
1161  // Split the vector length parameter.
1162  // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
1163  SDValue EVL = N->getOperand(3);
1164  EVT VecVT = N->getValueType(0);
1165  EVT EVLVT = EVL.getValueType();
1167  "Expecting the mask to be an evenly-sized vector");
1168  unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
1169  SDValue HalfNumElts =
1170  VecVT.isFixedLengthVector()
1171  ? DAG.getConstant(HalfMinNumElts, dl, EVLVT)
1172  : DAG.getVScale(dl, EVLVT,
1173  APInt(EVLVT.getScalarSizeInBits(), HalfMinNumElts));
1174  SDValue EVLLo = DAG.getNode(ISD::UMIN, dl, EVLVT, EVL, HalfNumElts);
1175  SDValue EVLHi = DAG.getNode(ISD::USUBSAT, dl, EVLVT, EVL, HalfNumElts);
1176 
1177  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1178  {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1179  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1180  {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1181 }
1182 
1183 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1184  SDValue &Hi) {
1185  SDValue Op0Lo, Op0Hi;
1186  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1187  SDValue Op1Lo, Op1Hi;
1188  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1189  SDValue Op2Lo, Op2Hi;
1190  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1191  SDLoc dl(N);
1192 
1193  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
1194  Op2Lo, N->getFlags());
1195  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
1196  Op2Hi, N->getFlags());
1197 }
1198 
1199 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1200  SDValue LHSLo, LHSHi;
1201  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1202  SDValue RHSLo, RHSHi;
1203  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1204  SDLoc dl(N);
1205  SDValue Op2 = N->getOperand(2);
1206 
1207  unsigned Opcode = N->getOpcode();
1208  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1209  N->getFlags());
1210  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1211  N->getFlags());
1212 }
1213 
1214 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1215  SDValue &Hi) {
1216  // We know the result is a vector. The input may be either a vector or a
1217  // scalar value.
1218  EVT LoVT, HiVT;
1219  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1220  SDLoc dl(N);
1221 
1222  SDValue InOp = N->getOperand(0);
1223  EVT InVT = InOp.getValueType();
1224 
1225  // Handle some special cases efficiently.
1226  switch (getTypeAction(InVT)) {
1234  break;
1237  // A scalar to vector conversion, where the scalar needs expansion.
1238  // If the vector is being split in two then we can just convert the
1239  // expanded pieces.
1240  if (LoVT == HiVT) {
1241  GetExpandedOp(InOp, Lo, Hi);
1242  if (DAG.getDataLayout().isBigEndian())
1243  std::swap(Lo, Hi);
1244  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1245  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1246  return;
1247  }
1248  break;
1250  // If the input is a vector that needs to be split, convert each split
1251  // piece of the input now.
1252  GetSplitVector(InOp, Lo, Hi);
1253  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1254  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1255  return;
1257  report_fatal_error("Scalarization of scalable vectors is not supported.");
1258  }
1259 
1260  // In the general case, convert the input to an integer and split it by hand.
1261  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1262  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1263  if (DAG.getDataLayout().isBigEndian())
1264  std::swap(LoIntVT, HiIntVT);
1265 
1266  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1267 
1268  if (DAG.getDataLayout().isBigEndian())
1269  std::swap(Lo, Hi);
1270  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1271  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1272 }
1273 
1274 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1275  SDValue &Hi) {
1276  EVT LoVT, HiVT;
1277  SDLoc dl(N);
1278  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1279  unsigned LoNumElts = LoVT.getVectorNumElements();
1280  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1281  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1282 
1283  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1284  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1285 }
1286 
1287 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1288  SDValue &Hi) {
1289  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1290  SDLoc dl(N);
1291  unsigned NumSubvectors = N->getNumOperands() / 2;
1292  if (NumSubvectors == 1) {
1293  Lo = N->getOperand(0);
1294  Hi = N->getOperand(1);
1295  return;
1296  }
1297 
1298  EVT LoVT, HiVT;
1299  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1300 
1301  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1302  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1303 
1304  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1305  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1306 }
1307 
1308 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1309  SDValue &Hi) {
1310  SDValue Vec = N->getOperand(0);
1311  SDValue Idx = N->getOperand(1);
1312  SDLoc dl(N);
1313 
1314  EVT LoVT, HiVT;
1315  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1316 
1317  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1318  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1319  Hi = DAG.getNode(
1320  ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1321  DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1322 }
1323 
1324 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1325  SDValue &Hi) {
1326  SDValue Vec = N->getOperand(0);
1327  SDValue SubVec = N->getOperand(1);
1328  SDValue Idx = N->getOperand(2);
1329  SDLoc dl(N);
1330  GetSplitVector(Vec, Lo, Hi);
1331 
1332  EVT VecVT = Vec.getValueType();
1333  EVT LoVT = Lo.getValueType();
1334  EVT SubVecVT = SubVec.getValueType();
1335  unsigned VecElems = VecVT.getVectorMinNumElements();
1336  unsigned SubElems = SubVecVT.getVectorMinNumElements();
1337  unsigned LoElems = LoVT.getVectorMinNumElements();
1338 
1339  // If we know the index is in the first half, and we know the subvector
1340  // doesn't cross the boundary between the halves, we can avoid spilling the
1341  // vector, and insert into the lower half of the split vector directly.
1342  unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1343  if (IdxVal + SubElems <= LoElems) {
1344  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1345  return;
1346  }
1347  // Similarly if the subvector is fully in the high half, but mind that we
1348  // can't tell whether a fixed-length subvector is fully within the high half
1349  // of a scalable vector.
1350  if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1351  IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1352  Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1353  DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1354  return;
1355  }
1356 
1357  // Spill the vector to the stack.
1358  // In cases where the vector is illegal it will be broken down into parts
1359  // and stored in parts - we should use the alignment for the smallest part.
1360  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1361  SDValue StackPtr =
1362  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1363  auto &MF = DAG.getMachineFunction();
1364  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1365  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1366 
1367  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1368  SmallestAlign);
1369 
1370  // Store the new subvector into the specified index.
1371  SDValue SubVecPtr =
1372  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1373  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1375 
1376  // Load the Lo part from the stack slot.
1377  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1378  SmallestAlign);
1379 
1380  // Increment the pointer to the other part.
1381  auto *Load = cast<LoadSDNode>(Lo);
1382  MachinePointerInfo MPI = Load->getPointerInfo();
1383  IncrementPointer(Load, LoVT, MPI, StackPtr);
1384 
1385  // Load the Hi part from the stack slot.
1386  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1387 }
1388 
1389 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1390  SDValue &Hi) {
1391  SDLoc dl(N);
1392  GetSplitVector(N->getOperand(0), Lo, Hi);
1393  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1394  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1395 }
1396 
1397 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1398  SDValue &Hi) {
1399  SDValue LHSLo, LHSHi;
1400  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1401  SDLoc DL(N);
1402 
1403  SDValue RHSLo, RHSHi;
1404  SDValue RHS = N->getOperand(1);
1405  EVT RHSVT = RHS.getValueType();
1406  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1407  GetSplitVector(RHS, RHSLo, RHSHi);
1408  else
1409  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1410 
1411 
1412  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1413  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1414 }
1415 
1416 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1417  SDValue &Hi) {
1418  SDValue LHSLo, LHSHi;
1419  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1420  SDLoc dl(N);
1421 
1422  EVT LoVT, HiVT;
1423  std::tie(LoVT, HiVT) =
1424  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1425 
1426  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1427  DAG.getValueType(LoVT));
1428  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1429  DAG.getValueType(HiVT));
1430 }
1431 
1432 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1433  SDValue &Hi) {
1434  unsigned Opcode = N->getOpcode();
1435  SDValue N0 = N->getOperand(0);
1436 
1437  SDLoc dl(N);
1438  SDValue InLo, InHi;
1439 
1440  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1441  GetSplitVector(N0, InLo, InHi);
1442  else
1443  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1444 
1445  EVT InLoVT = InLo.getValueType();
1446  unsigned InNumElements = InLoVT.getVectorNumElements();
1447 
1448  EVT OutLoVT, OutHiVT;
1449  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1450  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1451  assert((2 * OutNumElements) <= InNumElements &&
1452  "Illegal extend vector in reg split");
1453 
1454  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1455  // input vector (i.e. we only use InLo):
1456  // OutLo will extend the first OutNumElements from InLo.
1457  // OutHi will extend the next OutNumElements from InLo.
1458 
1459  // Shuffle the elements from InLo for OutHi into the bottom elements to
1460  // create a 'fake' InHi.
1461  SmallVector<int, 8> SplitHi(InNumElements, -1);
1462  for (unsigned i = 0; i != OutNumElements; ++i)
1463  SplitHi[i] = i + OutNumElements;
1464  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1465 
1466  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1467  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1468 }
1469 
1470 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1471  SDValue &Hi) {
1472  unsigned NumOps = N->getNumOperands();
1473  SDValue Chain = N->getOperand(0);
1474  EVT LoVT, HiVT;
1475  SDLoc dl(N);
1476  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1477 
1478  SmallVector<SDValue, 4> OpsLo(NumOps);
1479  SmallVector<SDValue, 4> OpsHi(NumOps);
1480 
1481  // The Chain is the first operand.
1482  OpsLo[0] = Chain;
1483  OpsHi[0] = Chain;
1484 
1485  // Now process the remaining operands.
1486  for (unsigned i = 1; i < NumOps; ++i) {
1487  SDValue Op = N->getOperand(i);
1488  SDValue OpLo = Op;
1489  SDValue OpHi = Op;
1490 
1491  EVT InVT = Op.getValueType();
1492  if (InVT.isVector()) {
1493  // If the input also splits, handle it directly for a
1494  // compile time speedup. Otherwise split it by hand.
1495  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1496  GetSplitVector(Op, OpLo, OpHi);
1497  else
1498  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1499  }
1500 
1501  OpsLo[i] = OpLo;
1502  OpsHi[i] = OpHi;
1503  }
1504 
1505  EVT LoValueVTs[] = {LoVT, MVT::Other};
1506  EVT HiValueVTs[] = {HiVT, MVT::Other};
1507  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1508  N->getFlags());
1509  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1510  N->getFlags());
1511 
1512  // Build a factor node to remember that this Op is independent of the
1513  // other one.
1514  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1515  Lo.getValue(1), Hi.getValue(1));
1516 
1517  // Legalize the chain result - switch anything that used the old chain to
1518  // use the new one.
1519  ReplaceValueWith(SDValue(N, 1), Chain);
1520 }
1521 
1522 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1523  SDValue Chain = N->getOperand(0);
1524  EVT VT = N->getValueType(0);
1525  unsigned NE = VT.getVectorNumElements();
1526  EVT EltVT = VT.getVectorElementType();
1527  SDLoc dl(N);
1528 
1529  SmallVector<SDValue, 8> Scalars;
1530  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1531 
1532  // If ResNE is 0, fully unroll the vector op.
1533  if (ResNE == 0)
1534  ResNE = NE;
1535  else if (NE > ResNE)
1536  NE = ResNE;
1537 
1538  //The results of each unrolled operation, including the chain.
1539  EVT ChainVTs[] = {EltVT, MVT::Other};
1540  SmallVector<SDValue, 8> Chains;
1541 
1542  unsigned i;
1543  for (i = 0; i != NE; ++i) {
1544  Operands[0] = Chain;
1545  for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1546  SDValue Operand = N->getOperand(j);
1547  EVT OperandVT = Operand.getValueType();
1548  if (OperandVT.isVector()) {
1549  EVT OperandEltVT = OperandVT.getVectorElementType();
1550  Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1551  Operand, DAG.getVectorIdxConstant(i, dl));
1552  } else {
1553  Operands[j] = Operand;
1554  }
1555  }
1556  SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1557  Scalar.getNode()->setFlags(N->getFlags());
1558 
1559  //Add in the scalar as well as its chain value to the
1560  //result vectors.
1561  Scalars.push_back(Scalar);
1562  Chains.push_back(Scalar.getValue(1));
1563  }
1564 
1565  for (; i < ResNE; ++i)
1566  Scalars.push_back(DAG.getUNDEF(EltVT));
1567 
1568  // Build a new factor node to connect the chain back together.
1569  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1570  ReplaceValueWith(SDValue(N, 1), Chain);
1571 
1572  // Create a new BUILD_VECTOR node
1573  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1574  return DAG.getBuildVector(VecVT, dl, Scalars);
1575 }
1576 
1577 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1578  SDValue &Lo, SDValue &Hi) {
1579  SDLoc dl(N);
1580  EVT ResVT = N->getValueType(0);
1581  EVT OvVT = N->getValueType(1);
1582  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1583  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1584  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1585 
1586  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1587  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1588  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1589  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1590  } else {
1591  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1592  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1593  }
1594 
1595  unsigned Opcode = N->getOpcode();
1596  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1597  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1598  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1599  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1600  LoNode->setFlags(N->getFlags());
1601  HiNode->setFlags(N->getFlags());
1602 
1603  Lo = SDValue(LoNode, ResNo);
1604  Hi = SDValue(HiNode, ResNo);
1605 
1606  // Replace the other vector result not being explicitly split here.
1607  unsigned OtherNo = 1 - ResNo;
1608  EVT OtherVT = N->getValueType(OtherNo);
1609  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1610  SetSplitVector(SDValue(N, OtherNo),
1611  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1612  } else {
1613  SDValue OtherVal = DAG.getNode(
1614  ISD::CONCAT_VECTORS, dl, OtherVT,
1615  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1616  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1617  }
1618 }
1619 
1620 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1621  SDValue &Hi) {
1622  SDValue Vec = N->getOperand(0);
1623  SDValue Elt = N->getOperand(1);
1624  SDValue Idx = N->getOperand(2);
1625  SDLoc dl(N);
1626  GetSplitVector(Vec, Lo, Hi);
1627 
1628  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1629  unsigned IdxVal = CIdx->getZExtValue();
1630  unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1631  if (IdxVal < LoNumElts) {
1632  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1633  Lo.getValueType(), Lo, Elt, Idx);
1634  return;
1635  } else if (!Vec.getValueType().isScalableVector()) {
1636  Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1637  DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1638  return;
1639  }
1640  }
1641 
1642  // See if the target wants to custom expand this node.
1643  if (CustomLowerNode(N, N->getValueType(0), true))
1644  return;
1645 
1646  // Make the vector elements byte-addressable if they aren't already.
1647  EVT VecVT = Vec.getValueType();
1648  EVT EltVT = VecVT.getVectorElementType();
1649  if (VecVT.getScalarSizeInBits() < 8) {
1650  EltVT = MVT::i8;
1651  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1652  VecVT.getVectorElementCount());
1653  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1654  // Extend the element type to match if needed.
1655  if (EltVT.bitsGT(Elt.getValueType()))
1656  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1657  }
1658 
1659  // Spill the vector to the stack.
1660  // In cases where the vector is illegal it will be broken down into parts
1661  // and stored in parts - we should use the alignment for the smallest part.
1662  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1663  SDValue StackPtr =
1664  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1665  auto &MF = DAG.getMachineFunction();
1666  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1667  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1668 
1669  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1670  SmallestAlign);
1671 
1672  // Store the new element. This may be larger than the vector element type,
1673  // so use a truncating store.
1674  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1675  Store = DAG.getTruncStore(
1676  Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1677  commonAlignment(SmallestAlign,
1678  EltVT.getFixedSizeInBits() / 8));
1679 
1680  EVT LoVT, HiVT;
1681  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1682 
1683  // Load the Lo part from the stack slot.
1684  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1685 
1686  // Increment the pointer to the other part.
1687  auto Load = cast<LoadSDNode>(Lo);
1688  MachinePointerInfo MPI = Load->getPointerInfo();
1689  IncrementPointer(Load, LoVT, MPI, StackPtr);
1690 
1691  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1692 
1693  // If we adjusted the original type, we need to truncate the results.
1694  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1695  if (LoVT != Lo.getValueType())
1696  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1697  if (HiVT != Hi.getValueType())
1698  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1699 }
1700 
1701 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1702  SDValue &Hi) {
1703  EVT LoVT, HiVT;
1704  SDLoc dl(N);
1705  assert(N->getValueType(0).isScalableVector() &&
1706  "Only scalable vectors are supported for STEP_VECTOR");
1707  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1708  SDValue Step = N->getOperand(0);
1709 
1710  Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1711 
1712  // Hi = Lo + (EltCnt * Step)
1713  EVT EltVT = Step.getValueType();
1714  APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1715  SDValue StartOfHi =
1716  DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1717  StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1718  StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1719 
1720  Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1721  Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1722 }
1723 
1724 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1725  SDValue &Hi) {
1726  EVT LoVT, HiVT;
1727  SDLoc dl(N);
1728  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1729  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1730  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1731  Hi = DAG.getUNDEF(HiVT);
1732  } else {
1733  assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1734  Hi = Lo;
1735  }
1736 }
1737 
1738 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1739  SDValue &Hi) {
1740  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1741  EVT LoVT, HiVT;
1742  SDLoc dl(LD);
1743  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1744 
1745  ISD::LoadExtType ExtType = LD->getExtensionType();
1746  SDValue Ch = LD->getChain();
1747  SDValue Ptr = LD->getBasePtr();
1748  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1749  EVT MemoryVT = LD->getMemoryVT();
1750  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1751  AAMDNodes AAInfo = LD->getAAInfo();
1752 
1753  EVT LoMemVT, HiMemVT;
1754  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1755 
1756  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1757  SDValue Value, NewChain;
1758  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1759  std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1760  ReplaceValueWith(SDValue(LD, 1), NewChain);
1761  return;
1762  }
1763 
1764  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1765  LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1766  MMOFlags, AAInfo);
1767 
1768  MachinePointerInfo MPI;
1769  IncrementPointer(LD, LoMemVT, MPI, Ptr);
1770 
1771  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1772  HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1773 
1774  // Build a factor node to remember that this load is independent of the
1775  // other one.
1776  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1777  Hi.getValue(1));
1778 
1779  // Legalize the chain result - switch anything that used the old chain to
1780  // use the new one.
1781  ReplaceValueWith(SDValue(LD, 1), Ch);
1782 }
1783 
1784 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1785  SDValue &Lo, SDValue &Hi) {
1786  assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1787  EVT LoVT, HiVT;
1788  SDLoc dl(MLD);
1789  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1790 
1791  SDValue Ch = MLD->getChain();
1792  SDValue Ptr = MLD->getBasePtr();
1793  SDValue Offset = MLD->getOffset();
1794  assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1795  SDValue Mask = MLD->getMask();
1796  SDValue PassThru = MLD->getPassThru();
1797  Align Alignment = MLD->getOriginalAlign();
1798  ISD::LoadExtType ExtType = MLD->getExtensionType();
1799 
1800  // Split Mask operand
1801  SDValue MaskLo, MaskHi;
1802  if (Mask.getOpcode() == ISD::SETCC) {
1803  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1804  } else {
1805  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1806  GetSplitVector(Mask, MaskLo, MaskHi);
1807  else
1808  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1809  }
1810 
1811  EVT MemoryVT = MLD->getMemoryVT();
1812  EVT LoMemVT, HiMemVT;
1813  bool HiIsEmpty = false;
1814  std::tie(LoMemVT, HiMemVT) =
1815  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1816 
1817  SDValue PassThruLo, PassThruHi;
1818  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1819  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1820  else
1821  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1822 
1825  MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
1826  MLD->getRanges());
1827 
1828  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1829  MMO, MLD->getAddressingMode(), ExtType,
1830  MLD->isExpandingLoad());
1831 
1832  if (HiIsEmpty) {
1833  // The hi masked load has zero storage size. We therefore simply set it to
1834  // the low masked load and rely on subsequent removal from the chain.
1835  Hi = Lo;
1836  } else {
1837  // Generate hi masked load.
1838  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1839  MLD->isExpandingLoad());
1840 
1841  MachinePointerInfo MPI;
1842  if (LoMemVT.isScalableVector())
1844  else
1845  MPI = MLD->getPointerInfo().getWithOffset(
1846  LoMemVT.getStoreSize().getFixedSize());
1847 
1850  MLD->getAAInfo(), MLD->getRanges());
1851 
1852  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1853  HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1854  MLD->isExpandingLoad());
1855  }
1856 
1857  // Build a factor node to remember that this load is independent of the
1858  // other one.
1859  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1860  Hi.getValue(1));
1861 
1862  // Legalize the chain result - switch anything that used the old chain to
1863  // use the new one.
1864  ReplaceValueWith(SDValue(MLD, 1), Ch);
1865 
1866 }
1867 
1868 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1869  SDValue &Lo, SDValue &Hi) {
1870  EVT LoVT, HiVT;
1871  SDLoc dl(MGT);
1872  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1873 
1874  SDValue Ch = MGT->getChain();
1875  SDValue Ptr = MGT->getBasePtr();
1876  SDValue Mask = MGT->getMask();
1877  SDValue PassThru = MGT->getPassThru();
1878  SDValue Index = MGT->getIndex();
1879  SDValue Scale = MGT->getScale();
1880  EVT MemoryVT = MGT->getMemoryVT();
1881  Align Alignment = MGT->getOriginalAlign();
1882  ISD::LoadExtType ExtType = MGT->getExtensionType();
1883 
1884  // Split Mask operand
1885  SDValue MaskLo, MaskHi;
1886  if (Mask.getOpcode() == ISD::SETCC) {
1887  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1888  } else {
1889  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1890  GetSplitVector(Mask, MaskLo, MaskHi);
1891  else
1892  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1893  }
1894 
1895  EVT LoMemVT, HiMemVT;
1896  // Split MemoryVT
1897  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1898 
1899  SDValue PassThruLo, PassThruHi;
1900  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1901  GetSplitVector(PassThru, PassThruLo, PassThruHi);
1902  else
1903  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1904 
1905  SDValue IndexHi, IndexLo;
1906  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1907  GetSplitVector(Index, IndexLo, IndexHi);
1908  else
1909  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1910 
1913  MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
1914  MGT->getRanges());
1915 
1916  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1917  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
1918  MMO, MGT->getIndexType(), ExtType);
1919 
1920  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1921  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
1922  MMO, MGT->getIndexType(), ExtType);
1923 
1924  // Build a factor node to remember that this load is independent of the
1925  // other one.
1926  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1927  Hi.getValue(1));
1928 
1929  // Legalize the chain result - switch anything that used the old chain to
1930  // use the new one.
1931  ReplaceValueWith(SDValue(MGT, 1), Ch);
1932 }
1933 
1934 
1935 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1936  assert(N->getValueType(0).isVector() &&
1937  N->getOperand(0).getValueType().isVector() &&
1938  "Operand types must be vectors");
1939 
1940  EVT LoVT, HiVT;
1941  SDLoc DL(N);
1942  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1943 
1944  // If the input also splits, handle it directly. Otherwise split it by hand.
1945  SDValue LL, LH, RL, RH;
1946  if (getTypeAction(N->getOperand(0).getValueType()) ==
1948  GetSplitVector(N->getOperand(0), LL, LH);
1949  else
1950  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1951 
1952  if (getTypeAction(N->getOperand(1).getValueType()) ==
1954  GetSplitVector(N->getOperand(1), RL, RH);
1955  else
1956  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1957 
1958  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1959  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1960 }
1961 
1962 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1963  SDValue &Hi) {
1964  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1965  EVT LoVT, HiVT;
1966  SDLoc dl(N);
1967  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1968 
1969  // If the input also splits, handle it directly for a compile time speedup.
1970  // Otherwise split it by hand.
1971  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1972  EVT InVT = N->getOperand(OpNo).getValueType();
1973  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1974  GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1975  else
1976  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1977 
1978  if (N->getOpcode() == ISD::FP_ROUND) {
1979  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
1980  N->getFlags());
1981  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
1982  N->getFlags());
1983  } else {
1984  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
1985  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
1986  }
1987 }
1988 
1989 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1990  SDValue &Hi) {
1991  SDLoc dl(N);
1992  EVT SrcVT = N->getOperand(0).getValueType();
1993  EVT DestVT = N->getValueType(0);
1994  EVT LoVT, HiVT;
1995  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1996 
1997  // We can do better than a generic split operation if the extend is doing
1998  // more than just doubling the width of the elements and the following are
1999  // true:
2000  // - The number of vector elements is even,
2001  // - the source type is legal,
2002  // - the type of a split source is illegal,
2003  // - the type of an extended (by doubling element size) source is legal, and
2004  // - the type of that extended source when split is legal.
2005  //
2006  // This won't necessarily completely legalize the operation, but it will
2007  // more effectively move in the right direction and prevent falling down
2008  // to scalarization in many cases due to the input vector being split too
2009  // far.
2010  if (SrcVT.getVectorElementCount().isKnownEven() &&
2011  SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2012  LLVMContext &Ctx = *DAG.getContext();
2013  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2014  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2015 
2016  EVT SplitLoVT, SplitHiVT;
2017  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2018  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2019  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2020  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2021  N->dump(&DAG); dbgs() << "\n");
2022  // Extend the source vector by one step.
2023  SDValue NewSrc =
2024  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2025  // Get the low and high halves of the new, extended one step, vector.
2026  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2027  // Extend those vector halves the rest of the way.
2028  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2029  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2030  return;
2031  }
2032  }
2033  // Fall back to the generic unary operator splitting otherwise.
2034  SplitVecRes_UnaryOp(N, Lo, Hi);
2035 }
2036 
2037 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2038  SDValue &Lo, SDValue &Hi) {
2039  // The low and high parts of the original input give four input vectors.
2040  SDValue Inputs[4];
2041  SDLoc dl(N);
2042  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2043  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2044  EVT NewVT = Inputs[0].getValueType();
2045  unsigned NewElts = NewVT.getVectorNumElements();
2046 
2047  // If Lo or Hi uses elements from at most two of the four input vectors, then
2048  // express it as a vector shuffle of those two inputs. Otherwise extract the
2049  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2051  for (unsigned High = 0; High < 2; ++High) {
2052  SDValue &Output = High ? Hi : Lo;
2053 
2054  // Build a shuffle mask for the output, discovering on the fly which
2055  // input vectors to use as shuffle operands (recorded in InputUsed).
2056  // If building a suitable shuffle vector proves too hard, then bail
2057  // out with useBuildVector set.
2058  unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
2059  unsigned FirstMaskIdx = High * NewElts;
2060  bool useBuildVector = false;
2061  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2062  // The mask element. This indexes into the input.
2063  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2064 
2065  // The input vector this mask element indexes into.
2066  unsigned Input = (unsigned)Idx / NewElts;
2067 
2068  if (Input >= array_lengthof(Inputs)) {
2069  // The mask element does not index into any input vector.
2070  Ops.push_back(-1);
2071  continue;
2072  }
2073 
2074  // Turn the index into an offset from the start of the input vector.
2075  Idx -= Input * NewElts;
2076 
2077  // Find or create a shuffle vector operand to hold this input.
2078  unsigned OpNo;
2079  for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
2080  if (InputUsed[OpNo] == Input) {
2081  // This input vector is already an operand.
2082  break;
2083  } else if (InputUsed[OpNo] == -1U) {
2084  // Create a new operand for this input vector.
2085  InputUsed[OpNo] = Input;
2086  break;
2087  }
2088  }
2089 
2090  if (OpNo >= array_lengthof(InputUsed)) {
2091  // More than two input vectors used! Give up on trying to create a
2092  // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
2093  useBuildVector = true;
2094  break;
2095  }
2096 
2097  // Add the mask index for the new shuffle vector.
2098  Ops.push_back(Idx + OpNo * NewElts);
2099  }
2100 
2101  if (useBuildVector) {
2102  EVT EltVT = NewVT.getVectorElementType();
2104 
2105  // Extract the input elements by hand.
2106  for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2107  // The mask element. This indexes into the input.
2108  int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2109 
2110  // The input vector this mask element indexes into.
2111  unsigned Input = (unsigned)Idx / NewElts;
2112 
2113  if (Input >= array_lengthof(Inputs)) {
2114  // The mask element is "undef" or indexes off the end of the input.
2115  SVOps.push_back(DAG.getUNDEF(EltVT));
2116  continue;
2117  }
2118 
2119  // Turn the index into an offset from the start of the input vector.
2120  Idx -= Input * NewElts;
2121 
2122  // Extract the vector element by hand.
2123  SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2124  Inputs[Input],
2125  DAG.getVectorIdxConstant(Idx, dl)));
2126  }
2127 
2128  // Construct the Lo/Hi output using a BUILD_VECTOR.
2129  Output = DAG.getBuildVector(NewVT, dl, SVOps);
2130  } else if (InputUsed[0] == -1U) {
2131  // No input vectors were used! The result is undefined.
2132  Output = DAG.getUNDEF(NewVT);
2133  } else {
2134  SDValue Op0 = Inputs[InputUsed[0]];
2135  // If only one input was used, use an undefined vector for the other.
2136  SDValue Op1 = InputUsed[1] == -1U ?
2137  DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
2138  // At least one input vector was used. Create a new shuffle vector.
2139  Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
2140  }
2141 
2142  Ops.clear();
2143  }
2144 }
2145 
2146 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2147  EVT OVT = N->getValueType(0);
2148  EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2149  SDValue Chain = N->getOperand(0);
2150  SDValue Ptr = N->getOperand(1);
2151  SDValue SV = N->getOperand(2);
2152  SDLoc dl(N);
2153 
2154  const Align Alignment =
2156 
2157  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2158  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2159  Chain = Hi.getValue(1);
2160 
2161  // Modified the chain - switch anything that used the old chain to use
2162  // the new one.
2163  ReplaceValueWith(SDValue(N, 1), Chain);
2164 }
2165 
2166 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2167  SDValue &Hi) {
2168  EVT DstVTLo, DstVTHi;
2169  std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2170  SDLoc dl(N);
2171 
2172  SDValue SrcLo, SrcHi;
2173  EVT SrcVT = N->getOperand(0).getValueType();
2174  if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2175  GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2176  else
2177  std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2178 
2179  Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2180  Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2181 }
2182 
2183 //===----------------------------------------------------------------------===//
2184 // Operand Vector Splitting
2185 //===----------------------------------------------------------------------===//
2186 
2187 /// This method is called when the specified operand of the specified node is
2188 /// found to need vector splitting. At this point, all of the result types of
2189 /// the node are known to be legal, but other operands of the node may need
2190 /// legalization as well as the specified one.
2191 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2192  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2193  SDValue Res = SDValue();
2194 
2195  // See if the target wants to custom split this node.
2196  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2197  return false;
2198 
2199  switch (N->getOpcode()) {
2200  default:
2201 #ifndef NDEBUG
2202  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2203  N->dump(&DAG);
2204  dbgs() << "\n";
2205 #endif
2206  report_fatal_error("Do not know how to split this operator's "
2207  "operand!\n");
2208 
2209  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2210  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2211  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2212  case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2213  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2214  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2215  case ISD::TRUNCATE:
2216  Res = SplitVecOp_TruncateHelper(N);
2217  break;
2218  case ISD::STRICT_FP_ROUND:
2219  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2220  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2221  case ISD::STORE:
2222  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2223  break;
2224  case ISD::MSTORE:
2225  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2226  break;
2227  case ISD::MSCATTER:
2228  Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
2229  break;
2230  case ISD::MGATHER:
2231  Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
2232  break;
2233  case ISD::VSELECT:
2234  Res = SplitVecOp_VSELECT(N, OpNo);
2235  break;
2238  case ISD::SINT_TO_FP:
2239  case ISD::UINT_TO_FP:
2240  if (N->getValueType(0).bitsLT(
2241  N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2242  Res = SplitVecOp_TruncateHelper(N);
2243  else
2244  Res = SplitVecOp_UnaryOp(N);
2245  break;
2246  case ISD::FP_TO_SINT_SAT:
2247  case ISD::FP_TO_UINT_SAT:
2248  Res = SplitVecOp_FP_TO_XINT_SAT(N);
2249  break;
2250  case ISD::FP_TO_SINT:
2251  case ISD::FP_TO_UINT:
2254  case ISD::STRICT_FP_EXTEND:
2255  case ISD::FP_EXTEND:
2256  case ISD::SIGN_EXTEND:
2257  case ISD::ZERO_EXTEND:
2258  case ISD::ANY_EXTEND:
2259  case ISD::FTRUNC:
2260  Res = SplitVecOp_UnaryOp(N);
2261  break;
2262 
2266  Res = SplitVecOp_ExtVecInRegOp(N);
2267  break;
2268 
2269  case ISD::VECREDUCE_FADD:
2270  case ISD::VECREDUCE_FMUL:
2271  case ISD::VECREDUCE_ADD:
2272  case ISD::VECREDUCE_MUL:
2273  case ISD::VECREDUCE_AND:
2274  case ISD::VECREDUCE_OR:
2275  case ISD::VECREDUCE_XOR:
2276  case ISD::VECREDUCE_SMAX:
2277  case ISD::VECREDUCE_SMIN:
2278  case ISD::VECREDUCE_UMAX:
2279  case ISD::VECREDUCE_UMIN:
2280  case ISD::VECREDUCE_FMAX:
2281  case ISD::VECREDUCE_FMIN:
2282  Res = SplitVecOp_VECREDUCE(N, OpNo);
2283  break;
2286  Res = SplitVecOp_VECREDUCE_SEQ(N);
2287  break;
2288  }
2289 
2290  // If the result is null, the sub-method took care of registering results etc.
2291  if (!Res.getNode()) return false;
2292 
2293  // If the result is N, the sub-method updated N in place. Tell the legalizer
2294  // core about this.
2295  if (Res.getNode() == N)
2296  return true;
2297 
2298  if (N->isStrictFPOpcode())
2299  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2300  "Invalid operand expansion");
2301  else
2302  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2303  "Invalid operand expansion");
2304 
2305  ReplaceValueWith(SDValue(N, 0), Res);
2306  return false;
2307 }
2308 
2309 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2310  // The only possibility for an illegal operand is the mask, since result type
2311  // legalization would have handled this node already otherwise.
2312  assert(OpNo == 0 && "Illegal operand must be mask");
2313 
2314  SDValue Mask = N->getOperand(0);
2315  SDValue Src0 = N->getOperand(1);
2316  SDValue Src1 = N->getOperand(2);
2317  EVT Src0VT = Src0.getValueType();
2318  SDLoc DL(N);
2319  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2320 
2321  SDValue Lo, Hi;
2322  GetSplitVector(N->getOperand(0), Lo, Hi);
2323  assert(Lo.getValueType() == Hi.getValueType() &&
2324  "Lo and Hi have differing types");
2325 
2326  EVT LoOpVT, HiOpVT;
2327  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2328  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2329 
2330  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2331  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2332  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2333  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2334 
2335  SDValue LoSelect =
2336  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2337  SDValue HiSelect =
2338  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2339 
2340  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2341 }
2342 
2343 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2344  EVT ResVT = N->getValueType(0);
2345  SDValue Lo, Hi;
2346  SDLoc dl(N);
2347 
2348  SDValue VecOp = N->getOperand(OpNo);
2349  EVT VecVT = VecOp.getValueType();
2350  assert(VecVT.isVector() && "Can only split reduce vector operand");
2351  GetSplitVector(VecOp, Lo, Hi);
2352  EVT LoOpVT, HiOpVT;
2353  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2354 
2355  // Use the appropriate scalar instruction on the split subvectors before
2356  // reducing the now partially reduced smaller vector.
2357  unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2358  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2359  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2360 }
2361 
2362 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2363  EVT ResVT = N->getValueType(0);
2364  SDValue Lo, Hi;
2365  SDLoc dl(N);
2366 
2367  SDValue AccOp = N->getOperand(0);
2368  SDValue VecOp = N->getOperand(1);
2369  SDNodeFlags Flags = N->getFlags();
2370 
2371  EVT VecVT = VecOp.getValueType();
2372  assert(VecVT.isVector() && "Can only split reduce vector operand");
2373  GetSplitVector(VecOp, Lo, Hi);
2374  EVT LoOpVT, HiOpVT;
2375  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2376 
2377  // Reduce low half.
2378  SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2379 
2380  // Reduce high half, using low half result as initial value.
2381  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2382 }
2383 
2384 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2385  // The result has a legal vector type, but the input needs splitting.
2386  EVT ResVT = N->getValueType(0);
2387  SDValue Lo, Hi;
2388  SDLoc dl(N);
2389  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2390  EVT InVT = Lo.getValueType();
2391 
2392  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2393  InVT.getVectorElementCount());
2394 
2395  if (N->isStrictFPOpcode()) {
2396  Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2397  { N->getOperand(0), Lo });
2398  Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2399  { N->getOperand(0), Hi });
2400 
2401  // Build a factor node to remember that this operation is independent
2402  // of the other one.
2403  SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2404  Hi.getValue(1));
2405 
2406  // Legalize the chain result - switch anything that used the old chain to
2407  // use the new one.
2408  ReplaceValueWith(SDValue(N, 1), Ch);
2409  } else {
2410  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2411  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2412  }
2413 
2414  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2415 }
2416 
2417 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2418  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2419  // end up being split all the way down to individual components. Convert the
2420  // split pieces into integers and reassemble.
2421  SDValue Lo, Hi;
2422  GetSplitVector(N->getOperand(0), Lo, Hi);
2423  Lo = BitConvertToInteger(Lo);
2424  Hi = BitConvertToInteger(Hi);
2425 
2426  if (DAG.getDataLayout().isBigEndian())
2427  std::swap(Lo, Hi);
2428 
2429  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2430  JoinIntegers(Lo, Hi));
2431 }
2432 
2433 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
2434  unsigned OpNo) {
2435  assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
2436  // We know that the result type is legal.
2437  EVT ResVT = N->getValueType(0);
2438 
2439  SDValue Vec = N->getOperand(0);
2440  SDValue SubVec = N->getOperand(1);
2441  SDValue Idx = N->getOperand(2);
2442  SDLoc dl(N);
2443 
2444  SDValue Lo, Hi;
2445  GetSplitVector(SubVec, Lo, Hi);
2446 
2447  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2448  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2449 
2450  SDValue FirstInsertion =
2451  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
2452  SDValue SecondInsertion =
2453  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
2454  DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
2455 
2456  return SecondInsertion;
2457 }
2458 
2459 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2460  // We know that the extracted result type is legal.
2461  EVT SubVT = N->getValueType(0);
2462 
2463  SDValue Idx = N->getOperand(1);
2464  SDLoc dl(N);
2465  SDValue Lo, Hi;
2466 
2467  if (SubVT.isScalableVector() !=
2468  N->getOperand(0).getValueType().isScalableVector())
2469  report_fatal_error("Extracting a fixed-length vector from an illegal "
2470  "scalable vector is not yet supported");
2471 
2472  GetSplitVector(N->getOperand(0), Lo, Hi);
2473 
2474  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2475  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2476 
2477  if (IdxVal < LoElts) {
2478  assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
2479  "Extracted subvector crosses vector split!");
2480  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2481  } else {
2482  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2483  DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
2484  }
2485 }
2486 
2487 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2488  SDValue Vec = N->getOperand(0);
2489  SDValue Idx = N->getOperand(1);
2490  EVT VecVT = Vec.getValueType();
2491 
2492  if (isa<ConstantSDNode>(Idx)) {
2493  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2494 
2495  SDValue Lo, Hi;
2496  GetSplitVector(Vec, Lo, Hi);
2497 
2498  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2499 
2500  if (IdxVal < LoElts)
2501  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2502  else if (!Vec.getValueType().isScalableVector())
2503  return SDValue(DAG.UpdateNodeOperands(N, Hi,
2504  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2505  Idx.getValueType())), 0);
2506  }
2507 
2508  // See if the target wants to custom expand this node.
2509  if (CustomLowerNode(N, N->getValueType(0), true))
2510  return SDValue();
2511 
2512  // Make the vector elements byte-addressable if they aren't already.
2513  SDLoc dl(N);
2514  EVT EltVT = VecVT.getVectorElementType();
2515  if (VecVT.getScalarSizeInBits() < 8) {
2516  EltVT = MVT::i8;
2517  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2518  VecVT.getVectorElementCount());
2519  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2520  }
2521 
2522  // Store the vector to the stack.
2523  // In cases where the vector is illegal it will be broken down into parts
2524  // and stored in parts - we should use the alignment for the smallest part.
2525  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2526  SDValue StackPtr =
2527  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2528  auto &MF = DAG.getMachineFunction();
2529  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2530  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2531  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2532  SmallestAlign);
2533 
2534  // Load back the required element.
2535  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2536 
2537  // FIXME: This is to handle i1 vectors with elements promoted to i8.
2538  // i1 vector handling needs general improvement.
2539  if (N->getValueType(0).bitsLT(EltVT)) {
2540  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2542  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2543  }
2544 
2545  return DAG.getExtLoad(
2546  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2548  commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
2549 }
2550 
2551 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2552  SDValue Lo, Hi;
2553 
2554  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2555  // splitting the result has the same effect as splitting the input operand.
2556  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2557 
2558  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2559 }
2560 
2561 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2562  unsigned OpNo) {
2563  EVT LoVT, HiVT;
2564  SDLoc dl(MGT);
2565  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2566 
2567  SDValue Ch = MGT->getChain();
2568  SDValue Ptr = MGT->getBasePtr();
2569  SDValue Index = MGT->getIndex();
2570  SDValue Scale = MGT->getScale();
2571  SDValue Mask = MGT->getMask();
2572  SDValue PassThru = MGT->getPassThru();
2573  Align Alignment = MGT->getOriginalAlign();
2574  ISD::LoadExtType ExtType = MGT->getExtensionType();
2575 
2576  SDValue MaskLo, MaskHi;
2577  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2578  // Split Mask operand
2579  GetSplitVector(Mask, MaskLo, MaskHi);
2580  else
2581  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2582 
2583  EVT MemoryVT = MGT->getMemoryVT();
2584  EVT LoMemVT, HiMemVT;
2585  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2586 
2587  SDValue PassThruLo, PassThruHi;
2588  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2589  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2590  else
2591  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2592 
2593  SDValue IndexHi, IndexLo;
2594  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2595  GetSplitVector(Index, IndexLo, IndexHi);
2596  else
2597  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2598 
2601  MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
2602  MGT->getRanges());
2603 
2604  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2605  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2606  OpsLo, MMO, MGT->getIndexType(), ExtType);
2607 
2608  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2609  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2610  OpsHi, MMO, MGT->getIndexType(), ExtType);
2611 
2612  // Build a factor node to remember that this load is independent of the
2613  // other one.
2614  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2615  Hi.getValue(1));
2616 
2617  // Legalize the chain result - switch anything that used the old chain to
2618  // use the new one.
2619  ReplaceValueWith(SDValue(MGT, 1), Ch);
2620 
2621  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2622  Hi);
2623  ReplaceValueWith(SDValue(MGT, 0), Res);
2624  return SDValue();
2625 }
2626 
2627 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2628  unsigned OpNo) {
2629  assert(N->isUnindexed() && "Indexed masked store of vector?");
2630  SDValue Ch = N->getChain();
2631  SDValue Ptr = N->getBasePtr();
2632  SDValue Offset = N->getOffset();
2633  assert(Offset.isUndef() && "Unexpected indexed masked store offset");
2634  SDValue Mask = N->getMask();
2635  SDValue Data = N->getValue();
2636  Align Alignment = N->getOriginalAlign();
2637  SDLoc DL(N);
2638 
2639  SDValue DataLo, DataHi;
2640  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2641  // Split Data operand
2642  GetSplitVector(Data, DataLo, DataHi);
2643  else
2644  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2645 
2646  // Split Mask operand
2647  SDValue MaskLo, MaskHi;
2648  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2649  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2650  } else {
2651  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2652  GetSplitVector(Mask, MaskLo, MaskHi);
2653  else
2654  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2655  }
2656 
2657  EVT MemoryVT = N->getMemoryVT();
2658  EVT LoMemVT, HiMemVT;
2659  bool HiIsEmpty = false;
2660  std::tie(LoMemVT, HiMemVT) =
2661  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
2662 
2663  SDValue Lo, Hi, Res;
2665  N->getPointerInfo(), MachineMemOperand::MOStore,
2666  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2667 
2668  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
2669  N->getAddressingMode(), N->isTruncatingStore(),
2670  N->isCompressingStore());
2671 
2672  if (HiIsEmpty) {
2673  // The hi masked store has zero storage size.
2674  // Only the lo masked store is needed.
2675  Res = Lo;
2676  } else {
2677 
2678  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2679  N->isCompressingStore());
2680 
2681  MachinePointerInfo MPI;
2682  if (LoMemVT.isScalableVector()) {
2683  Alignment = commonAlignment(
2684  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
2685  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
2686  } else
2687  MPI = N->getPointerInfo().getWithOffset(
2688  LoMemVT.getStoreSize().getFixedSize());
2689 
2692  N->getAAInfo(), N->getRanges());
2693 
2694  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
2695  N->getAddressingMode(), N->isTruncatingStore(),
2696  N->isCompressingStore());
2697 
2698  // Build a factor node to remember that this store is independent of the
2699  // other one.
2700  Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2701  }
2702 
2703  return Res;
2704 }
2705 
2706 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2707  unsigned OpNo) {
2708  SDValue Ch = N->getChain();
2709  SDValue Ptr = N->getBasePtr();
2710  SDValue Mask = N->getMask();
2711  SDValue Index = N->getIndex();
2712  SDValue Scale = N->getScale();
2713  SDValue Data = N->getValue();
2714  EVT MemoryVT = N->getMemoryVT();
2715  Align Alignment = N->getOriginalAlign();
2716  SDLoc DL(N);
2717 
2718  // Split all operands
2719 
2720  EVT LoMemVT, HiMemVT;
2721  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2722 
2723  SDValue DataLo, DataHi;
2724  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2725  // Split Data operand
2726  GetSplitVector(Data, DataLo, DataHi);
2727  else
2728  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2729 
2730  // Split Mask operand
2731  SDValue MaskLo, MaskHi;
2732  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2733  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2734  } else {
2735  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2736  GetSplitVector(Mask, MaskLo, MaskHi);
2737  else
2738  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2739  }
2740 
2741  SDValue IndexHi, IndexLo;
2742  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2743  GetSplitVector(Index, IndexLo, IndexHi);
2744  else
2745  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2746 
2747  SDValue Lo;
2749  N->getPointerInfo(), MachineMemOperand::MOStore,
2750  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2751 
2752  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2753  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
2754  DL, OpsLo, MMO, N->getIndexType(),
2755  N->isTruncatingStore());
2756 
2757  // The order of the Scatter operation after split is well defined. The "Hi"
2758  // part comes after the "Lo". So these two operations should be chained one
2759  // after another.
2760  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2761  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
2762  DL, OpsHi, MMO, N->getIndexType(),
2763  N->isTruncatingStore());
2764 }
2765 
2766 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2767  assert(N->isUnindexed() && "Indexed store of vector?");
2768  assert(OpNo == 1 && "Can only split the stored value");
2769  SDLoc DL(N);
2770 
2771  bool isTruncating = N->isTruncatingStore();
2772  SDValue Ch = N->getChain();
2773  SDValue Ptr = N->getBasePtr();
2774  EVT MemoryVT = N->getMemoryVT();
2775  Align Alignment = N->getOriginalAlign();
2776  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2777  AAMDNodes AAInfo = N->getAAInfo();
2778  SDValue Lo, Hi;
2779  GetSplitVector(N->getOperand(1), Lo, Hi);
2780 
2781  EVT LoMemVT, HiMemVT;
2782  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2783 
2784  // Scalarize if the split halves are not byte-sized.
2785  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2786  return TLI.scalarizeVectorStore(N, DAG);
2787 
2788  if (isTruncating)
2789  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2790  Alignment, MMOFlags, AAInfo);
2791  else
2792  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2793  AAInfo);
2794 
2795  MachinePointerInfo MPI;
2796  IncrementPointer(N, LoMemVT, MPI, Ptr);
2797 
2798  if (isTruncating)
2799  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
2800  HiMemVT, Alignment, MMOFlags, AAInfo);
2801  else
2802  Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
2803 
2804  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2805 }
2806 
2807 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2808  SDLoc DL(N);
2809 
2810  // The input operands all must have the same type, and we know the result
2811  // type is valid. Convert this to a buildvector which extracts all the
2812  // input elements.
2813  // TODO: If the input elements are power-two vectors, we could convert this to
2814  // a new CONCAT_VECTORS node with elements that are half-wide.
2816  EVT EltVT = N->getValueType(0).getVectorElementType();
2817  for (const SDValue &Op : N->op_values()) {
2818  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2819  i != e; ++i) {
2820  Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2821  DAG.getVectorIdxConstant(i, DL)));
2822  }
2823  }
2824 
2825  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2826 }
2827 
2828 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2829  // The result type is legal, but the input type is illegal. If splitting
2830  // ends up with the result type of each half still being legal, just
2831  // do that. If, however, that would result in an illegal result type,
2832  // we can try to get more clever with power-two vectors. Specifically,
2833  // split the input type, but also widen the result element size, then
2834  // concatenate the halves and truncate again. For example, consider a target
2835  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2836  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2837  // %inlo = v4i32 extract_subvector %in, 0
2838  // %inhi = v4i32 extract_subvector %in, 4
2839  // %lo16 = v4i16 trunc v4i32 %inlo
2840  // %hi16 = v4i16 trunc v4i32 %inhi
2841  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2842  // %res = v8i8 trunc v8i16 %in16
2843  //
2844  // Without this transform, the original truncate would end up being
2845  // scalarized, which is pretty much always a last resort.
2846  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
2847  SDValue InVec = N->getOperand(OpNo);
2848  EVT InVT = InVec->getValueType(0);
2849  EVT OutVT = N->getValueType(0);
2850  ElementCount NumElements = OutVT.getVectorElementCount();
2851  bool IsFloat = OutVT.isFloatingPoint();
2852 
2853  unsigned InElementSize = InVT.getScalarSizeInBits();
2854  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2855 
2856  // Determine the split output VT. If its legal we can just split dirctly.
2857  EVT LoOutVT, HiOutVT;
2858  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2859  assert(LoOutVT == HiOutVT && "Unequal split?");
2860 
2861  // If the input elements are only 1/2 the width of the result elements,
2862  // just use the normal splitting. Our trick only work if there's room
2863  // to split more than once.
2864  if (isTypeLegal(LoOutVT) ||
2865  InElementSize <= OutElementSize * 2)
2866  return SplitVecOp_UnaryOp(N);
2867  SDLoc DL(N);
2868 
2869  // Don't touch if this will be scalarized.
2870  EVT FinalVT = InVT;
2871  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2872  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2873 
2874  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2875  return SplitVecOp_UnaryOp(N);
2876 
2877  // Get the split input vector.
2878  SDValue InLoVec, InHiVec;
2879  GetSplitVector(InVec, InLoVec, InHiVec);
2880 
2881  // Truncate them to 1/2 the element size.
2882  //
2883  // This assumes the number of elements is a power of two; any vector that
2884  // isn't should be widened, not split.
2885  EVT HalfElementVT = IsFloat ?
2886  EVT::getFloatingPointVT(InElementSize/2) :
2887  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2888  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2889  NumElements.divideCoefficientBy(2));
2890 
2891  SDValue HalfLo;
2892  SDValue HalfHi;
2893  SDValue Chain;
2894  if (N->isStrictFPOpcode()) {
2895  HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2896  {N->getOperand(0), InLoVec});
2897  HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2898  {N->getOperand(0), InHiVec});
2899  // Legalize the chain result - switch anything that used the old chain to
2900  // use the new one.
2901  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
2902  HalfHi.getValue(1));
2903  } else {
2904  HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2905  HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2906  }
2907 
2908  // Concatenate them to get the full intermediate truncation result.
2909  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2910  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2911  HalfHi);
2912  // Now finish up by truncating all the way down to the original result
2913  // type. This should normally be something that ends up being legal directly,
2914  // but in theory if a target has very wide vectors and an annoyingly
2915  // restricted set of legal types, this split can chain to build things up.
2916 
2917  if (N->isStrictFPOpcode()) {
2918  SDValue Res = DAG.getNode(
2919  ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
2920  {Chain, InterVec,
2921  DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
2922  // Relink the chain
2923  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2924  return Res;
2925  }
2926 
2927  return IsFloat
2928  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2929  DAG.getTargetConstant(
2930  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2931  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2932 }
2933 
2934 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2935  assert(N->getValueType(0).isVector() &&
2936  N->getOperand(0).getValueType().isVector() &&
2937  "Operand types must be vectors");
2938  // The result has a legal vector type, but the input needs splitting.
2939  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2940  SDLoc DL(N);
2941  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2942  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2943  auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
2944 
2945  LLVMContext &Context = *DAG.getContext();
2946  EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
2947  EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
2948 
2949  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2950  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2951  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2952 
2953  EVT OpVT = N->getOperand(0).getValueType();
2954  ISD::NodeType ExtendCode =
2956  return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2957 }
2958 
2959 
2960 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2961  // The result has a legal vector type, but the input needs splitting.
2962  EVT ResVT = N->getValueType(0);
2963  SDValue Lo, Hi;
2964  SDLoc DL(N);
2965  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2966  EVT InVT = Lo.getValueType();
2967 
2968  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2969  InVT.getVectorElementCount());
2970 
2971  if (N->isStrictFPOpcode()) {
2972  Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2973  { N->getOperand(0), Lo, N->getOperand(2) });
2974  Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2975  { N->getOperand(0), Hi, N->getOperand(2) });
2976  // Legalize the chain result - switch anything that used the old chain to
2977  // use the new one.
2978  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2979  Lo.getValue(1), Hi.getValue(1));
2980  ReplaceValueWith(SDValue(N, 1), NewChain);
2981  } else {
2982  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2983  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2984  }
2985 
2986  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2987 }
2988 
2989 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2990  // The result (and the first input) has a legal vector type, but the second
2991  // input needs splitting.
2992  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2993 }
2994 
2995 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
2996  EVT ResVT = N->getValueType(0);
2997  SDValue Lo, Hi;
2998  SDLoc dl(N);
2999  GetSplitVector(N->getOperand(0), Lo, Hi);
3000  EVT InVT = Lo.getValueType();
3001 
3002  EVT NewResVT =
3004  InVT.getVectorElementCount());
3005 
3006  Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3007  Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3008 
3009  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3010 }
3011 
3012 //===----------------------------------------------------------------------===//
3013 // Result Vector Widening
3014 //===----------------------------------------------------------------------===//
3015 
3016 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3017  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3018  dbgs() << "\n");
3019 
3020  // See if the target wants to custom widen this node.
3021  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3022  return;
3023 
3024  SDValue Res = SDValue();
3025  switch (N->getOpcode()) {
3026  default:
3027 #ifndef NDEBUG
3028  dbgs() << "WidenVectorResult #" << ResNo << ": ";
3029  N->dump(&DAG);
3030  dbgs() << "\n";
3031 #endif
3032  llvm_unreachable("Do not know how to widen the result of this operator!");
3033 
3034  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3035  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3036  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3037  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3038  case ISD::INSERT_SUBVECTOR:
3039  Res = WidenVecRes_INSERT_SUBVECTOR(N);
3040  break;
3041  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3042  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3043  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3044  case ISD::SPLAT_VECTOR:
3045  case ISD::SCALAR_TO_VECTOR:
3046  Res = WidenVecRes_ScalarOp(N);
3047  break;
3048  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3049  case ISD::VSELECT:
3050  case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
3051  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3052  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3053  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3054  case ISD::VECTOR_SHUFFLE:
3055  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3056  break;
3057  case ISD::MLOAD:
3058  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3059  break;
3060  case ISD::MGATHER:
3061  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3062  break;
3063 
3064  case ISD::ADD:
3065  case ISD::AND:
3066  case ISD::MUL:
3067  case ISD::MULHS:
3068  case ISD::MULHU:
3069  case ISD::OR:
3070  case ISD::SUB:
3071  case ISD::XOR:
3072  case ISD::SHL:
3073  case ISD::SRA:
3074  case ISD::SRL:
3075  case ISD::FMINNUM:
3076  case ISD::FMAXNUM:
3077  case ISD::FMINIMUM:
3078  case ISD::FMAXIMUM:
3079  case ISD::SMIN:
3080  case ISD::SMAX:
3081  case ISD::UMIN:
3082  case ISD::UMAX:
3083  case ISD::UADDSAT:
3084  case ISD::SADDSAT:
3085  case ISD::USUBSAT:
3086  case ISD::SSUBSAT:
3087  case ISD::SSHLSAT:
3088  case ISD::USHLSAT:
3089  case ISD::ROTL:
3090  case ISD::ROTR:
3091  Res = WidenVecRes_Binary(N, /*IsVP*/ false);
3092  break;
3093 
3094  case ISD::FADD:
3095  case ISD::FMUL:
3096  case ISD::FPOW:
3097  case ISD::FSUB:
3098  case ISD::FDIV:
3099  case ISD::FREM:
3100  case ISD::SDIV:
3101  case ISD::UDIV:
3102  case ISD::SREM:
3103  case ISD::UREM:
3104  Res = WidenVecRes_BinaryCanTrap(N);
3105  break;
3106 
3107  case ISD::SMULFIX:
3108  case ISD::SMULFIXSAT:
3109  case ISD::UMULFIX:
3110  case ISD::UMULFIXSAT:
3111  // These are binary operations, but with an extra operand that shouldn't
3112  // be widened (the scale).
3113  Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3114  break;
3115 
3116 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3117  case ISD::STRICT_##DAGN:
3118 #include "llvm/IR/ConstrainedOps.def"
3119  Res = WidenVecRes_StrictFP(N);
3120  break;
3121 
3122  case ISD::UADDO:
3123  case ISD::SADDO:
3124  case ISD::USUBO:
3125  case ISD::SSUBO:
3126  case ISD::UMULO:
3127  case ISD::SMULO:
3128  Res = WidenVecRes_OverflowOp(N, ResNo);
3129  break;
3130 
3131  case ISD::FCOPYSIGN:
3132  Res = WidenVecRes_FCOPYSIGN(N);
3133  break;
3134 
3135  case ISD::FPOWI:
3136  Res = WidenVecRes_POWI(N);
3137  break;
3138 
3142  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
3143  break;
3144 
3145  case ISD::ANY_EXTEND:
3146  case ISD::FP_EXTEND:
3147  case ISD::FP_ROUND:
3148  case ISD::FP_TO_SINT:
3149  case ISD::FP_TO_UINT:
3150  case ISD::SIGN_EXTEND:
3151  case ISD::SINT_TO_FP:
3152  case ISD::TRUNCATE:
3153  case ISD::UINT_TO_FP:
3154  case ISD::ZERO_EXTEND:
3155  Res = WidenVecRes_Convert(N);
3156  break;
3157 
3158  case ISD::FP_TO_SINT_SAT:
3159  case ISD::FP_TO_UINT_SAT:
3160  Res = WidenVecRes_FP_TO_XINT_SAT(N);
3161  break;
3162 
3163  case ISD::FABS:
3164  case ISD::FCEIL:
3165  case ISD::FCOS:
3166  case ISD::FEXP:
3167  case ISD::FEXP2:
3168  case ISD::FFLOOR:
3169  case ISD::FLOG:
3170  case ISD::FLOG10:
3171  case ISD::FLOG2:
3172  case ISD::FNEARBYINT:
3173  case ISD::FRINT:
3174  case ISD::FROUND:
3175  case ISD::FROUNDEVEN:
3176  case ISD::FSIN:
3177  case ISD::FSQRT:
3178  case ISD::FTRUNC: {
3179  // We're going to widen this vector op to a legal type by padding with undef
3180  // elements. If the wide vector op is eventually going to be expanded to
3181  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3182  // libcalls on the undef elements.
3183  EVT VT = N->getValueType(0);
3184  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3185  if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3186  TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3187  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3188  break;
3189  }
3190  }
3191  // If the target has custom/legal support for the scalar FP intrinsic ops
3192  // (they are probably not destined to become libcalls), then widen those like
3193  // any other unary ops.
3195 
3196  case ISD::ABS:
3197  case ISD::BITREVERSE:
3198  case ISD::BSWAP:
3199  case ISD::CTLZ:
3200  case ISD::CTLZ_ZERO_UNDEF:
3201  case ISD::CTPOP:
3202  case ISD::CTTZ:
3203  case ISD::CTTZ_ZERO_UNDEF:
3204  case ISD::FNEG:
3205  case ISD::FREEZE:
3206  case ISD::ARITH_FENCE:
3207  case ISD::FCANONICALIZE:
3208  Res = WidenVecRes_Unary(N);
3209  break;
3210  case ISD::FMA:
3211  case ISD::FSHL:
3212  case ISD::FSHR:
3213  Res = WidenVecRes_Ternary(N);
3214  break;
3215  case ISD::VP_ADD:
3216  case ISD::VP_AND:
3217  case ISD::VP_MUL:
3218  case ISD::VP_OR:
3219  case ISD::VP_SUB:
3220  case ISD::VP_XOR:
3221  case ISD::VP_SHL:
3222  case ISD::VP_LSHR:
3223  case ISD::VP_ASHR:
3224  case ISD::VP_SDIV:
3225  case ISD::VP_UDIV:
3226  case ISD::VP_SREM:
3227  case ISD::VP_UREM:
3228  case ISD::VP_FADD:
3229  case ISD::VP_FSUB:
3230  case ISD::VP_FMUL:
3231  case ISD::VP_FDIV:
3232  case ISD::VP_FREM:
3233  // Vector-predicated binary op widening. Note that -- unlike the
3234  // unpredicated versions -- we don't have to worry about trapping on
3235  // operations like UDIV, FADD, etc., as we pass on the original vector
3236  // length parameter. This means the widened elements containing garbage
3237  // aren't active.
3238  Res = WidenVecRes_Binary(N, /*IsVP*/ true);
3239  break;
3240  }
3241 
3242  // If Res is null, the sub-method took care of registering the result.
3243  if (Res.getNode())
3244  SetWidenedVector(SDValue(N, ResNo), Res);
3245 }
3246 
3247 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
3248  // Ternary op widening.
3249  SDLoc dl(N);
3250  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3251  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3252  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3253  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
3254  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
3255 }
3256 
3257 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N, bool IsVP) {
3258  // Binary op widening.
3259  SDLoc dl(N);
3260  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3261  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3262  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3263  if (!IsVP)
3264  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
3265  N->getFlags());
3266  // For VP operations, we must also widen the mask. Note that the mask type
3267  // may not actually need widening, leading it be split along with the VP
3268  // operation.
3269  // FIXME: This could lead to an infinite split/widen loop. We only handle the
3270  // case where the mask needs widening to an identically-sized type as the
3271  // vector inputs.
3272  SDValue Mask = N->getOperand(2);
3273  assert(getTypeAction(Mask.getValueType()) ==
3275  "Unable to widen binary VP op");
3276  Mask = GetWidenedVector(Mask);
3277  assert(Mask.getValueType().getVectorElementCount() ==
3278  WidenVT.getVectorElementCount() &&
3279  "Unable to widen binary VP op");
3280  return DAG.getNode(N->getOpcode(), dl, WidenVT,
3281  {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
3282 }
3283 
3284 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3285  // Binary op widening, but with an extra operand that shouldn't be widened.
3286  SDLoc dl(N);
3287  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3288  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3289  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3290  SDValue InOp3 = N->getOperand(2);
3291  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3292  N->getFlags());
3293 }
3294 
3295 // Given a vector of operations that have been broken up to widen, see
3296 // if we can collect them together into the next widest legal VT. This
3297 // implementation is trap-safe.
3299  SmallVectorImpl<SDValue> &ConcatOps,
3300  unsigned ConcatEnd, EVT VT, EVT MaxVT,
3301  EVT WidenVT) {
3302  // Check to see if we have a single operation with the widen type.
3303  if (ConcatEnd == 1) {
3304  VT = ConcatOps[0].getValueType();
3305  if (VT == WidenVT)
3306  return ConcatOps[0];
3307  }
3308 
3309  SDLoc dl(ConcatOps[0]);
3310  EVT WidenEltVT = WidenVT.getVectorElementType();
3311 
3312  // while (Some element of ConcatOps is not of type MaxVT) {
3313  // From the end of ConcatOps, collect elements of the same type and put
3314  // them into an op of the next larger supported type
3315  // }
3316  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3317  int Idx = ConcatEnd - 1;
3318  VT = ConcatOps[Idx--].getValueType();
3319  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3320  Idx--;
3321 
3322  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3323  EVT NextVT;
3324  do {
3325  NextSize *= 2;
3326  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3327  } while (!TLI.isTypeLegal(NextVT));
3328 
3329  if (!VT.isVector()) {
3330  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3331  SDValue VecOp = DAG.getUNDEF(NextVT);
3332  unsigned NumToInsert = ConcatEnd - Idx - 1;
3333  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3334  VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3335  ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3336  }
3337  ConcatOps[Idx+1] = VecOp;
3338  ConcatEnd = Idx + 2;
3339  } else {
3340  // Vector type, create a CONCAT_VECTORS of type NextVT
3341  SDValue undefVec = DAG.getUNDEF(VT);
3342  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3343  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3344  unsigned RealVals = ConcatEnd - Idx - 1;
3345  unsigned SubConcatEnd = 0;
3346  unsigned SubConcatIdx = Idx + 1;
3347  while (SubConcatEnd < RealVals)
3348  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3349  while (SubConcatEnd < OpsToConcat)
3350  SubConcatOps[SubConcatEnd++] = undefVec;
3351  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3352  NextVT, SubConcatOps);
3353  ConcatEnd = SubConcatIdx + 1;
3354  }
3355  }
3356 
3357  // Check to see if we have a single operation with the widen type.
3358  if (ConcatEnd == 1) {
3359  VT = ConcatOps[0].getValueType();
3360  if (VT == WidenVT)
3361  return ConcatOps[0];
3362  }
3363 
3364  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
3365  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
3366  if (NumOps != ConcatEnd ) {
3367  SDValue UndefVal = DAG.getUNDEF(MaxVT);
3368  for (unsigned j = ConcatEnd; j < NumOps; ++j)
3369  ConcatOps[j] = UndefVal;
3370  }
3371  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3372  makeArrayRef(ConcatOps.data(), NumOps));
3373 }
3374 
3375 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3376  // Binary op widening for operations that can trap.
3377  unsigned Opcode = N->getOpcode();
3378  SDLoc dl(N);
3379  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3380  EVT WidenEltVT = WidenVT.getVectorElementType();
3381  EVT VT = WidenVT;
3382  unsigned NumElts = VT.getVectorNumElements();
3383  const SDNodeFlags Flags = N->getFlags();
3384  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3385  NumElts = NumElts / 2;
3386  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3387  }
3388 
3389  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
3390  // Operation doesn't trap so just widen as normal.
3391  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3392  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3393  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3394  }
3395 
3396  // No legal vector version so unroll the vector operation and then widen.
3397  if (NumElts == 1)
3398  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3399 
3400  // Since the operation can trap, apply operation on the original vector.
3401  EVT MaxVT = VT;
3402  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3403  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3404  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3405 
3406  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3407  unsigned ConcatEnd = 0; // Current ConcatOps index.
3408  int Idx = 0; // Current Idx into input vectors.
3409 
3410  // NumElts := greatest legal vector size (at most WidenVT)
3411  // while (orig. vector has unhandled elements) {
3412  // take munches of size NumElts from the beginning and add to ConcatOps
3413  // NumElts := next smaller supported vector size or 1
3414  // }
3415  while (CurNumElts != 0) {
3416  while (CurNumElts >= NumElts) {
3417  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3418  DAG.getVectorIdxConstant(Idx, dl));
3419  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3420  DAG.getVectorIdxConstant(Idx, dl));
3421  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3422  Idx += NumElts;
3423  CurNumElts -= NumElts;
3424  }
3425  do {
3426  NumElts = NumElts / 2;
3427  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3428  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3429 
3430  if (NumElts == 1) {
3431  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3432  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3433  InOp1, DAG.getVectorIdxConstant(Idx, dl));
3434  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3435  InOp2, DAG.getVectorIdxConstant(Idx, dl));
3436  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3437  EOp1, EOp2, Flags);
3438  }
3439  CurNumElts = 0;
3440  }
3441  }
3442 
3443  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3444 }
3445 
3446 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3447  switch (N->getOpcode()) {
3448  case ISD::STRICT_FSETCC:
3449  case ISD::STRICT_FSETCCS:
3450  return WidenVecRes_STRICT_FSETCC(N);
3451  case ISD::STRICT_FP_EXTEND:
3452  case ISD::STRICT_FP_ROUND:
3457  return WidenVecRes_Convert_StrictFP(N);
3458  default:
3459  break;
3460  }
3461 
3462  // StrictFP op widening for operations that can trap.
3463  unsigned NumOpers = N->getNumOperands();
3464  unsigned Opcode = N->getOpcode();
3465  SDLoc dl(N);
3466  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3467  EVT WidenEltVT = WidenVT.getVectorElementType();
3468  EVT VT = WidenVT;
3469  unsigned NumElts = VT.getVectorNumElements();
3470  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3471  NumElts = NumElts / 2;
3472  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3473  }
3474 
3475  // No legal vector version so unroll the vector operation and then widen.
3476  if (NumElts == 1)
3477  return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3478 
3479  // Since the operation can trap, apply operation on the original vector.
3480  EVT MaxVT = VT;
3482  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3483 
3484  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3485  SmallVector<SDValue, 16> Chains;
3486  unsigned ConcatEnd = 0; // Current ConcatOps index.
3487  int Idx = 0; // Current Idx into input vectors.
3488 
3489  // The Chain is the first operand.
3490  InOps.push_back(N->getOperand(0));
3491 
3492  // Now process the remaining operands.
3493  for (unsigned i = 1; i < NumOpers; ++i) {
3494  SDValue Oper = N->getOperand(i);
3495 
3496  if (Oper.getValueType().isVector()) {
3497  assert(Oper.getValueType() == N->getValueType(0) &&
3498  "Invalid operand type to widen!");
3499  Oper = GetWidenedVector(Oper);
3500  }
3501 
3502  InOps.push_back(Oper);
3503  }
3504 
3505  // NumElts := greatest legal vector size (at most WidenVT)
3506  // while (orig. vector has unhandled elements) {
3507  // take munches of size NumElts from the beginning and add to ConcatOps
3508  // NumElts := next smaller supported vector size or 1
3509  // }
3510  while (CurNumElts != 0) {
3511  while (CurNumElts >= NumElts) {
3513 
3514  for (unsigned i = 0; i < NumOpers; ++i) {
3515  SDValue Op = InOps[i];
3516 
3517  if (Op.getValueType().isVector())
3518  Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3519  DAG.getVectorIdxConstant(Idx, dl));
3520 
3521  EOps.push_back(Op);
3522  }
3523 
3524  EVT OperVT[] = {VT, MVT::Other};
3525  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3526  ConcatOps[ConcatEnd++] = Oper;
3527  Chains.push_back(Oper.getValue(1));
3528  Idx += NumElts;
3529  CurNumElts -= NumElts;
3530  }
3531  do {
3532  NumElts = NumElts / 2;
3533  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3534  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3535 
3536  if (NumElts == 1) {
3537  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3539 
3540  for (unsigned i = 0; i < NumOpers; ++i) {
3541  SDValue Op = InOps[i];
3542 
3543  if (Op.getValueType().isVector())
3544  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3545  DAG.getVectorIdxConstant(Idx, dl));
3546 
3547  EOps.push_back(Op);
3548  }
3549 
3550  EVT WidenVT[] = {WidenEltVT, MVT::Other};
3551  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3552  ConcatOps[ConcatEnd++] = Oper;
3553  Chains.push_back(Oper.getValue(1));
3554  }
3555  CurNumElts = 0;
3556  }
3557  }
3558 
3559  // Build a factor node to remember all the Ops that have been created.
3560  SDValue NewChain;
3561  if (Chains.size() == 1)
3562  NewChain = Chains[0];
3563  else
3564  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3565  ReplaceValueWith(SDValue(N, 1), NewChain);
3566 
3567  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3568 }
3569 
3570 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3571  SDLoc DL(N);
3572  EVT ResVT = N->getValueType(0);
3573  EVT OvVT = N->getValueType(1);
3574  EVT WideResVT, WideOvVT;
3575  SDValue WideLHS, WideRHS;
3576 
3577  // TODO: This might result in a widen/split loop.
3578  if (ResNo == 0) {
3579  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3580  WideOvVT = EVT::getVectorVT(
3581  *DAG.getContext(), OvVT.getVectorElementType(),
3582  WideResVT.getVectorNumElements());
3583 
3584  WideLHS = GetWidenedVector(N->getOperand(0));
3585  WideRHS = GetWidenedVector(N->getOperand(1));
3586  } else {
3587  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3588  WideResVT = EVT::getVectorVT(
3589  *DAG.getContext(), ResVT.getVectorElementType(),
3590  WideOvVT.getVectorNumElements());
3591 
3592  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3593  WideLHS = DAG.getNode(
3594  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3595  N->getOperand(0), Zero);
3596  WideRHS = DAG.getNode(
3597  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3598  N->getOperand(1), Zero);
3599  }
3600 
3601  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3602  SDNode *WideNode = DAG.getNode(
3603  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3604 
3605  // Replace the other vector result not being explicitly widened here.
3606  unsigned OtherNo = 1 - ResNo;
3607  EVT OtherVT = N->getValueType(OtherNo);
3608  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3609  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3610  } else {
3611  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3612  SDValue OtherVal = DAG.getNode(
3613  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3614  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3615  }
3616 
3617  return SDValue(WideNode, ResNo);
3618 }
3619 
3620 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3621  LLVMContext &Ctx = *DAG.getContext();
3622  SDValue InOp = N->getOperand(0);
3623  SDLoc DL(N);
3624 
3625  EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
3626  ElementCount WidenEC = WidenVT.getVectorElementCount();
3627 
3628  EVT InVT = InOp.getValueType();
3629 
3630  unsigned Opcode = N->getOpcode();
3631  const SDNodeFlags Flags = N->getFlags();
3632 
3633  // Handle the case of ZERO_EXTEND where the promoted InVT element size does
3634  // not equal that of WidenVT.
3635  if (N->getOpcode() == ISD::ZERO_EXTEND &&
3636  getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
3637  TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
3638  WidenVT.getScalarSizeInBits()) {
3639  InOp = ZExtPromotedInteger(InOp);
3640  InVT = InOp.getValueType();
3641  if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
3642  Opcode = ISD::TRUNCATE;
3643  }
3644 
3645  EVT InEltVT = InVT.getVectorElementType();
3646  EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
3647  ElementCount InVTEC = InVT.getVectorElementCount();
3648 
3649  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3650  InOp = GetWidenedVector(N->getOperand(0));
3651  InVT = InOp.getValueType();
3652  InVTEC = InVT.getVectorElementCount();
3653  if (InVTEC == WidenEC) {
3654  if (N->getNumOperands() == 1)
3655  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3656  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3657  }
3658  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3659  // If both input and result vector types are of same width, extend
3660  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3661  // accepts fewer elements in the result than in the input.
3662  if (Opcode == ISD::ANY_EXTEND)
3663  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3664  if (Opcode == ISD::SIGN_EXTEND)
3665  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3666  if (Opcode == ISD::ZERO_EXTEND)
3667  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3668  }
3669  }
3670 
3671  if (TLI.isTypeLegal(InWidenVT)) {
3672  // Because the result and the input are different vector types, widening
3673  // the result could create a legal type but widening the input might make
3674  // it an illegal type that might lead to repeatedly splitting the input
3675  // and then widening it. To avoid this, we widen the input only if
3676  // it results in a legal type.
3677  if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
3678  // Widen the input and call convert on the widened input vector.
3679  unsigned NumConcat =
3680  WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
3681  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3682  Ops[0] = InOp;
3683  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3684  if (N->getNumOperands() == 1)
3685  return DAG.getNode(Opcode, DL, WidenVT, InVec);
3686  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3687  }
3688 
3689  if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
3690  SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3691  DAG.getVectorIdxConstant(0, DL));
3692  // Extract the input and convert the shorten input vector.
3693  if (N->getNumOperands() == 1)
3694  return DAG.getNode(Opcode, DL, WidenVT, InVal);
3695  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3696  }
3697  }
3698 
3699  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3700  EVT EltVT = WidenVT.getVectorElementType();
3701  SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT));
3702  // Use the original element count so we don't do more scalar opts than
3703  // necessary.
3704  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3705  for (unsigned i=0; i < MinElts; ++i) {
3706  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3707  DAG.getVectorIdxConstant(i, DL));
3708  if (N->getNumOperands() == 1)
3709  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3710  else
3711  Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3712  }
3713 
3714  return DAG.getBuildVector(WidenVT, DL, Ops);
3715 }
3716 
3717 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
3718  SDLoc dl(N);
3719  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3720  ElementCount WidenNumElts = WidenVT.getVectorElementCount();
3721 
3722  SDValue Src = N->getOperand(0);
3723  EVT SrcVT = Src.getValueType();
3724 
3725  // Also widen the input.
3726  if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
3727  Src = GetWidenedVector(Src);
3728  SrcVT = Src.getValueType();
3729  }
3730 
3731  // Input and output not widened to the same size, give up.
3732  if (WidenNumElts != SrcVT.getVectorElementCount())
3733  return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
3734 
3735  return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
3736 }
3737 
3738 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3739  SDValue InOp = N->getOperand(1);
3740  SDLoc DL(N);
3741  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3742 
3743  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3744  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3745 
3746  EVT InVT = InOp.getValueType();
3747  EVT InEltVT = InVT.getVectorElementType();
3748 
3749  unsigned Opcode = N->getOpcode();
3750 
3751  // FIXME: Optimizations need to be implemented here.
3752 
3753  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3754  EVT EltVT = WidenVT.getVectorElementType();
3755  std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
3756  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3757  SmallVector<SDValue, 32> OpChains;
3758  // Use the original element count so we don't do more scalar opts than
3759  // necessary.
3760  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3761  for (unsigned i=0; i < MinElts; ++i) {
3762  NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3763  DAG.getVectorIdxConstant(i, DL));
3764  Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3765  OpChains.push_back(Ops[i].getValue(1));
3766  }
3767  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3768  ReplaceValueWith(SDValue(N, 1), NewChain);
3769 
3770  return DAG.getBuildVector(WidenVT, DL, Ops);
3771 }
3772 
3773 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3774  unsigned Opcode = N->getOpcode();
3775  SDValue InOp = N->getOperand(0);
3776  SDLoc DL(N);
3777 
3778  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3779  EVT WidenSVT = WidenVT.getVectorElementType();
3780  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3781 
3782  EVT InVT = InOp.getValueType();
3783  EVT InSVT = InVT.getVectorElementType();
3784  unsigned InVTNumElts = InVT.getVectorNumElements();
3785 
3786  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3787  InOp = GetWidenedVector(InOp);
3788  InVT = InOp.getValueType();
3789  if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3790  switch (Opcode) {
3794  return DAG.getNode(Opcode, DL, WidenVT, InOp);
3795  }
3796  }
3797  }
3798 
3799  // Unroll, extend the scalars and rebuild the vector.
3801  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3802  SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3803  DAG.getVectorIdxConstant(i, DL));
3804  switch (Opcode) {
3806  Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3807  break;
3809  Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3810  break;
3812  Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3813  break;
3814  default:
3815  llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3816  }
3817  Ops.push_back(Val);
3818  }
3819 
3820  while (Ops.size() != WidenNumElts)
3821  Ops.push_back(DAG.getUNDEF(WidenSVT));
3822 
3823  return DAG.getBuildVector(WidenVT, DL, Ops);
3824 }
3825 
3826 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3827  // If this is an FCOPYSIGN with same input types, we can treat it as a
3828  // normal (can trap) binary op.
3829  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3830  return WidenVecRes_BinaryCanTrap(N);
3831 
3832  // If the types are different, fall back to unrolling.
3833  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3834  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3835 }
3836 
3837 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3838  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3839  SDValue InOp = GetWidenedVector(N->getOperand(0));
3840  SDValue ShOp = N->getOperand(1);
3841  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3842 }
3843 
3844 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3845  // Unary op widening.
3846  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3847  SDValue InOp = GetWidenedVector(N->getOperand(0));
3848  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3849 }
3850 
3851 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3852  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3853  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3854  cast<VTSDNode>(N->getOperand(1))->getVT()
3855  .getVectorElementType(),
3856  WidenVT.getVectorNumElements());
3857  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3858  return DAG.getNode(N->getOpcode(), SDLoc(N),
3859  WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3860 }
3861 
3862 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3863  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3864  return GetWidenedVector(WidenVec);
3865 }
3866 
3867 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3868  SDValue InOp = N->getOperand(0);
3869  EVT InVT = InOp.getValueType();
3870  EVT VT = N->getValueType(0);
3871  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3872  SDLoc dl(N);
3873 
3874  switch (getTypeAction(InVT)) {
3876  break;
3878  report_fatal_error("Scalarization of scalable vectors is not supported.");
3880  // If the incoming type is a vector that is being promoted, then
3881  // we know that the elements are arranged differently and that we
3882  // must perform the conversion using a stack slot.
3883  if (InVT.isVector())
3884  break;
3885 
3886  // If the InOp is promoted to the same size, convert it. Otherwise,
3887  // fall out of the switch and widen the promoted input.
3888  SDValue NInOp = GetPromotedInteger(InOp);
3889  EVT NInVT = NInOp.getValueType();
3890  if (WidenVT.bitsEq(NInVT)) {
3891  // For big endian targets we need to shift the input integer or the
3892  // interesting bits will end up at the wrong place.
3893  if (DAG.getDataLayout().isBigEndian()) {
3894  unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
3895  EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
3896  assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
3897  NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
3898  DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
3899  }
3900  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
3901  }
3902  InOp = NInOp;
3903  InVT = NInVT;
3904  break;
3905  }
3913  break;
3915  // If the InOp is widened to the same size, convert it. Otherwise, fall
3916  // out of the switch and widen the widened input.
3917  InOp = GetWidenedVector(InOp);
3918  InVT = InOp.getValueType();
3919  if (WidenVT.bitsEq(InVT))
3920  // The input widens to the same size. Convert to the widen value.
3921  return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3922  break;
3923  }
3924 
3925  unsigned WidenSize = WidenVT.getSizeInBits();
3926  unsigned InSize = InVT.getSizeInBits();
3927  // x86mmx is not an acceptable vector element type, so don't try.
3928  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3929  // Determine new input vector type. The new input vector type will use
3930  // the same element type (if its a vector) or use the input type as a
3931  // vector. It is the same size as the type to widen to.
3932  EVT NewInVT;
3933  unsigned NewNumElts = WidenSize / InSize;
3934  if (InVT.isVector()) {
3935  EVT InEltVT = InVT.getVectorElementType();
3936  NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3937  WidenSize / InEltVT.getSizeInBits());
3938  } else {
3939  NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3940  }
3941 
3942  if (TLI.isTypeLegal(NewInVT)) {
3943  SDValue NewVec;
3944  if (InVT.isVector()) {
3945  // Because the result and the input are different vector types, widening
3946  // the result could create a legal type but widening the input might make
3947  // it an illegal type that might lead to repeatedly splitting the input
3948  // and then widening it. To avoid this, we widen the input only if
3949  // it results in a legal type.
3950  SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3951  Ops[0] = InOp;
3952 
3953  NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3954  } else {
3955  NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3956  }
3957  return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3958  }
3959  }
3960 
3961  return CreateStackStoreLoad(InOp, WidenVT);
3962 }
3963 
3964 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3965  SDLoc dl(N);
3966  // Build a vector with undefined for the new nodes.
3967  EVT VT = N->getValueType(0);
3968 
3969  // Integer BUILD_VECTOR operands may be larger than the node's vector element
3970  // type. The UNDEFs need to have the same type as the existing operands.
3971  EVT EltVT = N->getOperand(0).getValueType();
3972  unsigned NumElts = VT.getVectorNumElements();
3973 
3974  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3975  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3976 
3977  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3978  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3979  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3980 
3981  return DAG.getBuildVector(WidenVT, dl, NewOps);
3982 }
3983 
3984 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3985  EVT InVT = N->getOperand(0).getValueType();
3986  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3987  SDLoc dl(N);
3988  unsigned NumOperands = N->getNumOperands();
3989 
3990  bool InputWidened = false; // Indicates we need to widen the input.
3991  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3992  unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
3993  unsigned NumInElts = InVT.getVectorMinNumElements();
3994  if (WidenNumElts % NumInElts == 0) {
3995  // Add undef vectors to widen to correct length.
3996  unsigned NumConcat = WidenNumElts / NumInElts;
3997  SDValue UndefVal = DAG.getUNDEF(InVT);
3998  SmallVector<SDValue, 16> Ops(NumConcat);
3999  for (unsigned i=0; i < NumOperands; ++i)
4000  Ops[i] = N->getOperand(i);
4001  for (unsigned i = NumOperands; i != NumConcat; ++i)
4002  Ops[i] = UndefVal;
4003  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
4004  }
4005  } else {
4006  InputWidened = true;
4007  if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4008  // The inputs and the result are widen to the same value.
4009  unsigned i;
4010  for (i=1; i < NumOperands; ++i)
4011  if (!N->getOperand(i).isUndef())
4012  break;
4013 
4014  if (i == NumOperands)
4015  // Everything but the first operand is an UNDEF so just return the
4016  // widened first operand.
4017  return GetWidenedVector(N->getOperand(0));
4018 
4019  if (NumOperands == 2) {
4020  assert(!WidenVT.isScalableVector() &&
4021  "Cannot use vector shuffles to widen CONCAT_VECTOR result");
4022  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4023  unsigned NumInElts = InVT.getVectorNumElements();
4024 
4025  // Replace concat of two operands with a shuffle.
4026  SmallVector<int, 16> MaskOps(WidenNumElts, -1);
4027  for (unsigned i = 0; i < NumInElts; ++i) {
4028  MaskOps[i] = i;
4029  MaskOps[i + NumInElts] = i + WidenNumElts;
4030  }
4031  return DAG.getVectorShuffle(WidenVT, dl,
4032  GetWidenedVector(N->getOperand(0)),
4033  GetWidenedVector(N->getOperand(1)),
4034  MaskOps);
4035  }
4036  }
4037  }
4038 
4039  assert(!WidenVT.isScalableVector() &&
4040  "Cannot use build vectors to widen CONCAT_VECTOR result");
4041  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4042  unsigned NumInElts = InVT.getVectorNumElements();
4043 
4044  // Fall back to use extracts and build vector.
4045  EVT EltVT = WidenVT.getVectorElementType();
4046  SmallVector<SDValue, 16> Ops(WidenNumElts);
4047  unsigned Idx = 0;
4048  for (unsigned i=0; i < NumOperands; ++i) {
4049  SDValue InOp = N->getOperand(i);
4050  if (InputWidened)
4051  InOp = GetWidenedVector(InOp);
4052  for (unsigned j = 0; j < NumInElts; ++j)
4053  Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4054  DAG.getVectorIdxConstant(j, dl));
4055  }
4056  SDValue UndefVal = DAG.getUNDEF(EltVT);
4057  for (; Idx < WidenNumElts; ++Idx)
4058  Ops[Idx] = UndefVal;
4059  return DAG.getBuildVector(WidenVT, dl, Ops);
4060 }
4061 
4062 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
4063  EVT VT = N->getValueType(0);
4064  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4065  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4066  SDValue InOp2 = N->getOperand(1);
4067  SDValue Idx = N->getOperand(2);
4068  SDLoc dl(N);
4069  return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WidenVT, InOp1, InOp2, Idx);
4070 }
4071 
4072 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
4073  EVT VT = N->getValueType(0);
4074  EVT EltVT = VT.getVectorElementType();
4075  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4076  SDValue InOp = N->getOperand(0);
4077  SDValue Idx = N->getOperand(1);
4078  SDLoc dl(N);
4079 
4080  auto InOpTypeAction = getTypeAction(InOp.getValueType());
4081  if (InOpTypeAction == TargetLowering::TypeWidenVector)
4082  InOp = GetWidenedVector(InOp);
4083 
4084  EVT InVT = InOp.getValueType();
4085 
4086  // Check if we can just return the input vector after widening.
4087  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
4088  if (IdxVal == 0 && InVT == WidenVT)
4089  return InOp;
4090 
4091  // Check if we can extract from the vector.
4092  unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
4093  unsigned InNumElts = InVT.getVectorMinNumElements();
4094  if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
4095  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
4096 
4097  if (VT.isScalableVector()) {
4098  // Try to split the operation up into smaller extracts and concat the
4099  // results together, e.g.
4100  // nxv6i64 extract_subvector(nxv12i64, 6)
4101  // <->
4102  // nxv8i64 concat(
4103  // nxv2i64 extract_subvector(nxv16i64, 6)
4104  // nxv2i64 extract_subvector(nxv16i64, 8)
4105  // nxv2i64 extract_subvector(nxv16i64, 10)
4106  // undef)
4107  unsigned VTNElts = VT.getVectorMinNumElements();
4108  unsigned GCD = greatestCommonDivisor(VTNElts, WidenNumElts);
4109  assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
4110  "down type's element count");
4111  EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4113  // Avoid recursion around e.g. nxv1i8.
4114  if (getTypeAction(PartVT) != TargetLowering::TypeWidenVector) {
4115  SmallVector<SDValue> Parts;
4116  unsigned I = 0;
4117  for (; I < VTNElts / GCD; ++I)
4118  Parts.push_back(
4119  DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, PartVT, InOp,
4120  DAG.getVectorIdxConstant(IdxVal + I * GCD, dl)));
4121  for (; I < WidenNumElts / GCD; ++I)
4122  Parts.push_back(DAG.getUNDEF(PartVT));
4123 
4124  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts);
4125  }
4126 
4127  report_fatal_error("Don't know how to widen the result of "
4128  "EXTRACT_SUBVECTOR for scalable vectors");
4129  }
4130 
4131  // We could try widening the input to the right length but for now, extract
4132  // the original elements, fill the rest with undefs and build a vector.
4133  SmallVector<SDValue, 16> Ops(WidenNumElts);
4134  unsigned NumElts = VT.getVectorNumElements();
4135  unsigned i;
4136  for (i = 0; i < NumElts; ++i)
4137  Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4138  DAG.getVectorIdxConstant(IdxVal + i, dl));
4139 
4140  SDValue UndefVal = DAG.getUNDEF(EltVT);
4141  for (; i < WidenNumElts; ++i)
4142  Ops[i] = UndefVal;
4143  return DAG.getBuildVector(WidenVT, dl, Ops);
4144 }
4145 
4146 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
4147  SDValue InOp = GetWidenedVector(N->getOperand(0));
4148  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
4149  InOp.getValueType(), InOp,
4150  N->getOperand(1), N->getOperand(2));
4151 }
4152 
4153 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
4154  LoadSDNode *LD = cast<LoadSDNode>(N);
4155  ISD::LoadExtType ExtType = LD->getExtensionType();
4156 
4157  // A vector must always be stored in memory as-is, i.e. without any padding
4158  // between the elements, since various code depend on it, e.g. in the
4159  // handling of a bitcast of a vector type to int, which may be done with a
4160  // vector store followed by an integer load. A vector that does not have
4161  // elements that are byte-sized must therefore be stored as an integer
4162  // built out of the extracted vector elements.
4163  if (!LD->getMemoryVT().isByteSized()) {
4164  SDValue Value, NewChain;
4165  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
4166  ReplaceValueWith(SDValue(LD, 0), Value);
4167  ReplaceValueWith(SDValue(LD, 1), NewChain);
4168  return SDValue();
4169  }
4170 
4171  SDValue Result;
4172  SmallVector<SDValue, 16> LdChain; // Chain for the series of load
4173  if (ExtType != ISD::NON_EXTLOAD)
4174  Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
4175  else
4176  Result = GenWidenVectorLoads(LdChain, LD);
4177 
4178  if (Result) {
4179  // If we generate a single load, we can use that for the chain. Otherwise,
4180  // build a factor node to remember the multiple loads are independent and
4181  // chain to that.
4182  SDValue NewChain;
4183  if (LdChain.size() == 1)
4184  NewChain = LdChain[0];
4185  else
4186  NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
4187 
4188  // Modified the chain - switch anything that used the old chain to use
4189  // the new one.
4190  ReplaceValueWith(SDValue(N, 1), NewChain);
4191 
4192  return Result;
4193  }
4194 
4195  // Generate a vector-predicated load if it is custom/legal on the target. To
4196  // avoid possible recursion, only do this if the widened mask type is legal.
4197  // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
4198  // removed from the IR by the ExpandVectorPredication pass but we're
4199  // reintroducing them here.
4200  EVT LdVT = LD->getMemoryVT();
4201  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), LdVT);
4202  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4203  WideVT.getVectorElementCount());
4204  if (ExtType == ISD::NON_EXTLOAD && WideVT.isScalableVector() &&
4205  TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WideVT) &&
4206  TLI.isTypeLegal(WideMaskVT)) {
4207  SDLoc DL(N);
4208  SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT);
4209  MVT EVLVT = TLI.getVPExplicitVectorLengthTy();
4210  unsigned NumVTElts = LdVT.getVectorMinNumElements();
4211  SDValue EVL =
4212  DAG.getVScale(DL, EVLVT, APInt(EVLVT.getScalarSizeInBits(), NumVTElts));
4213  const auto *MMO = LD->getMemOperand();
4214  SDValue NewLoad =
4215  DAG.getLoadVP(WideVT, DL, LD->getChain(), LD->getBasePtr(), Mask, EVL,
4216  MMO->getPointerInfo(), MMO->getAlign(), MMO->getFlags(),
4217  MMO->getAAInfo());
4218 
4219  // Modified the chain - switch anything that used the old chain to use
4220  // the new one.
4221  ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1));
4222 
4223  return NewLoad;
4224  }
4225 
4226  report_fatal_error("Unable to widen vector load");
4227 }
4228 
4229 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
4230 
4231  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
4232  SDValue Mask = N->getMask();
4233  EVT MaskVT = Mask.getValueType();
4234  SDValue PassThru = GetWidenedVector(N->getPassThru());
4235  ISD::LoadExtType ExtType = N->getExtensionType();
4236  SDLoc dl(N);
4237 
4238  // The mask should be widened as well
4239  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4240  MaskVT.getVectorElementType(),
4241  WidenVT.getVectorNumElements());
4242  Mask = ModifyToType(Mask, WideMaskVT, true);
4243 
4244  SDValue Res = DAG.getMaskedLoad(
4245  WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
4246  PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
4247  ExtType, N->isExpandingLoad());
4248  // Legalize the chain result - switch anything that used the old chain to
4249  // use the new one.
4250  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4251  return Res;
4252 }
4253 
4254 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
4255 
4256  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4257  SDValue Mask = N->getMask();
4258  EVT MaskVT = Mask.getValueType();
4259  SDValue PassThru = GetWidenedVector(N->getPassThru());
4260  SDValue Scale = N->getScale();
4261  unsigned NumElts = WideVT.getVectorNumElements();
4262  SDLoc dl(N);
4263 
4264  // The mask should be widened as well
4265  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4266  MaskVT.getVectorElementType(),
4267  WideVT.getVectorNumElements());
4268  Mask = ModifyToType(Mask, WideMaskVT, true);
4269 
4270  // Widen the Index operand
4271  SDValue Index = N->getIndex();
4272  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4273  Index.getValueType().getScalarType(),
4274  NumElts);
4275  Index = ModifyToType(Index, WideIndexVT);
4276  SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
4277  Scale };
4278 
4279  // Widen the MemoryType
4280  EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4281  N->getMemoryVT().getScalarType(), NumElts);
4282  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
4283  WideMemVT, dl, Ops, N->getMemOperand(),
4284  N->getIndexType(), N->getExtensionType());
4285 
4286  // Legalize the chain result - switch anything that used the old chain to
4287  // use the new one.
4288  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4289  return Res;
4290 }
4291 
4292 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
4293  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4294  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
4295 }
4296 
4297 // Return true is this is a SETCC node or a strict version of it.
4298 static inline bool isSETCCOp(unsigned Opcode) {
4299  switch (Opcode) {
4300  case ISD::SETCC:
4301  case ISD::STRICT_FSETCC:
4302  case ISD::STRICT_FSETCCS:
4303  return true;
4304  }
4305  return false;
4306 }
4307 
4308 // Return true if this is a node that could have two SETCCs as operands.
4309 static inline bool isLogicalMaskOp(unsigned Opcode) {
4310  switch (Opcode) {
4311  case ISD::AND:
4312  case ISD::OR:
4313  case ISD::XOR:
4314  return true;
4315  }
4316  return false;
4317 }
4318 
4319 // If N is a SETCC or a strict variant of it, return the type
4320 // of the compare operands.
4322  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4323  return N->getOperand(OpNo).getValueType();
4324 }
4325 
4326 // This is used just for the assert in convertMask(). Check that this either
4327 // a SETCC or a previously handled SETCC by convertMask().
4328 #ifndef NDEBUG
4329 static inline bool isSETCCorConvertedSETCC(SDValue N) {
4330  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
4331  N = N.getOperand(0);
4332  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
4333  for (unsigned i = 1; i < N->getNumOperands(); ++i)
4334  if (!N->getOperand(i)->isUndef())
4335  return false;
4336  N = N.getOperand(0);
4337  }
4338 
4339  if (N.getOpcode() == ISD::TRUNCATE)
4340  N = N.getOperand(0);
4341  else if (N.getOpcode() == ISD::SIGN_EXTEND)
4342  N = N.getOperand(0);
4343 
4344  if (isLogicalMaskOp(N.getOpcode()))
4345  return isSETCCorConvertedSETCC(N.getOperand(0)) &&
4346  isSETCCorConvertedSETCC(N.getOperand(1));
4347 
4348  return (isSETCCOp(N.getOpcode()) ||
4350 }
4351 #endif
4352 
4353 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
4354 // to ToMaskVT if needed with vector extension or truncation.
4355 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
4356  EVT ToMaskVT) {
4357  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
4358  // FIXME: This code seems to be too restrictive, we might consider
4359  // generalizing it or dropping it.
4360  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
4361 
4362  // Make a new Mask node, with a legal result VT.
4363  SDValue Mask;
4365  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
4366  Ops.push_back(InMask->getOperand(i));
4367  if (InMask->isStrictFPOpcode()) {
4368  Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
4369  { MaskVT, MVT::Other }, Ops);
4370  ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
4371  }
4372  else
4373  Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
4374 
4375  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
4376  // extend or truncate is needed.
4377  LLVMContext &Ctx = *DAG.getContext();
4378  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
4379  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
4380  if (MaskScalarBits < ToMaskScalBits) {
4381  EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4382  MaskVT.getVectorNumElements());
4383  Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
4384  } else if (MaskScalarBits > ToMaskScalBits) {
4385  EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4386  MaskVT.getVectorNumElements());
4387  Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
4388  }
4389 
4390  assert(Mask->getValueType(0).getScalarSizeInBits() ==
4391  ToMaskVT.getScalarSizeInBits() &&
4392  "Mask should have the right element size by now.");
4393 
4394  // Adjust Mask to the right number of elements.
4395  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
4396  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
4397  SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
4398  Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
4399  ZeroIdx);
4400  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
4401  unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
4402  EVT SubVT = Mask->getValueType(0);
4403  SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
4404  SubOps[0] = Mask;
4405  Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
4406  }
4407 
4408  assert((Mask->getValueType(0) == ToMaskVT) &&
4409  "A mask of ToMaskVT should have been produced by now.");
4410 
4411  return Mask;
4412 }
4413 
4414 // This method tries to handle some special cases for the vselect mask
4415 // and if needed adjusting the mask vector type to match that of the VSELECT.
4416 // Without it, many cases end up with scalarization of the SETCC, with many
4417 // unnecessary instructions.
4418 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
4419  LLVMContext &Ctx = *DAG.getContext();
4420  SDValue