LLVM  16.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "LegalizeTypes.h"
26 #include "llvm/IR/DataLayout.h"
28 #include "llvm/Support/TypeSize.h"
30 #include <numeric>
31 
32 using namespace llvm;
33 
34 #define DEBUG_TYPE "legalize-types"
35 
36 //===----------------------------------------------------------------------===//
37 // Result Vector Scalarization: <1 x ty> -> ty.
38 //===----------------------------------------------------------------------===//
39 
40 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
41  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
42  dbgs() << "\n");
43  SDValue R = SDValue();
44 
45  switch (N->getOpcode()) {
46  default:
47 #ifndef NDEBUG
48  dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
49  N->dump(&DAG);
50  dbgs() << "\n";
51 #endif
52  report_fatal_error("Do not know how to scalarize the result of this "
53  "operator!\n");
54 
55  case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
56  case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
57  case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
58  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
59  case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
60  case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
61  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
62  case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
63  case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
64  case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
65  case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
66  case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
67  case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
68  case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
69  case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
70  case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71  case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
75  R = ScalarizeVecRes_VecInregOp(N);
76  break;
77  case ISD::ABS:
78  case ISD::ANY_EXTEND:
79  case ISD::BITREVERSE:
80  case ISD::BSWAP:
81  case ISD::CTLZ:
83  case ISD::CTPOP:
84  case ISD::CTTZ:
86  case ISD::FABS:
87  case ISD::FCEIL:
88  case ISD::FCOS:
89  case ISD::FEXP:
90  case ISD::FEXP2:
91  case ISD::FFLOOR:
92  case ISD::FLOG:
93  case ISD::FLOG10:
94  case ISD::FLOG2:
95  case ISD::FNEARBYINT:
96  case ISD::FNEG:
97  case ISD::FREEZE:
98  case ISD::ARITH_FENCE:
99  case ISD::FP_EXTEND:
100  case ISD::FP_TO_SINT:
101  case ISD::FP_TO_UINT:
102  case ISD::FRINT:
103  case ISD::FROUND:
104  case ISD::FROUNDEVEN:
105  case ISD::FSIN:
106  case ISD::FSQRT:
107  case ISD::FTRUNC:
108  case ISD::SIGN_EXTEND:
109  case ISD::SINT_TO_FP:
110  case ISD::TRUNCATE:
111  case ISD::UINT_TO_FP:
112  case ISD::ZERO_EXTEND:
113  case ISD::FCANONICALIZE:
114  R = ScalarizeVecRes_UnaryOp(N);
115  break;
116 
117  case ISD::ADD:
118  case ISD::AND:
119  case ISD::FADD:
120  case ISD::FCOPYSIGN:
121  case ISD::FDIV:
122  case ISD::FMUL:
123  case ISD::FMINNUM:
124  case ISD::FMAXNUM:
125  case ISD::FMINNUM_IEEE:
126  case ISD::FMAXNUM_IEEE:
127  case ISD::FMINIMUM:
128  case ISD::FMAXIMUM:
129  case ISD::SMIN:
130  case ISD::SMAX:
131  case ISD::UMIN:
132  case ISD::UMAX:
133 
134  case ISD::SADDSAT:
135  case ISD::UADDSAT:
136  case ISD::SSUBSAT:
137  case ISD::USUBSAT:
138  case ISD::SSHLSAT:
139  case ISD::USHLSAT:
140 
141  case ISD::FPOW:
142  case ISD::FREM:
143  case ISD::FSUB:
144  case ISD::MUL:
145  case ISD::OR:
146  case ISD::SDIV:
147  case ISD::SREM:
148  case ISD::SUB:
149  case ISD::UDIV:
150  case ISD::UREM:
151  case ISD::XOR:
152  case ISD::SHL:
153  case ISD::SRA:
154  case ISD::SRL:
155  case ISD::ROTL:
156  case ISD::ROTR:
157  R = ScalarizeVecRes_BinOp(N);
158  break;
159  case ISD::FMA:
160  case ISD::FSHL:
161  case ISD::FSHR:
162  R = ScalarizeVecRes_TernaryOp(N);
163  break;
164 
165 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
166  case ISD::STRICT_##DAGN:
167 #include "llvm/IR/ConstrainedOps.def"
168  R = ScalarizeVecRes_StrictFPOp(N);
169  break;
170 
171  case ISD::FP_TO_UINT_SAT:
172  case ISD::FP_TO_SINT_SAT:
173  R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
174  break;
175 
176  case ISD::UADDO:
177  case ISD::SADDO:
178  case ISD::USUBO:
179  case ISD::SSUBO:
180  case ISD::UMULO:
181  case ISD::SMULO:
182  R = ScalarizeVecRes_OverflowOp(N, ResNo);
183  break;
184  case ISD::SMULFIX:
185  case ISD::SMULFIXSAT:
186  case ISD::UMULFIX:
187  case ISD::UMULFIXSAT:
188  case ISD::SDIVFIX:
189  case ISD::SDIVFIXSAT:
190  case ISD::UDIVFIX:
191  case ISD::UDIVFIXSAT:
192  R = ScalarizeVecRes_FIX(N);
193  break;
194  }
195 
196  // If R is null, the sub-method took care of registering the result.
197  if (R.getNode())
198  SetScalarizedVector(SDValue(N, ResNo), R);
199 }
200 
201 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
202  SDValue LHS = GetScalarizedVector(N->getOperand(0));
203  SDValue RHS = GetScalarizedVector(N->getOperand(1));
204  return DAG.getNode(N->getOpcode(), SDLoc(N),
205  LHS.getValueType(), LHS, RHS, N->getFlags());
206 }
207 
208 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
209  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
210  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
211  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
212  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
213  Op2, N->getFlags());
214 }
215 
216 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
217  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
218  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
219  SDValue Op2 = N->getOperand(2);
220  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
221  Op2, N->getFlags());
222 }
223 
224 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
225  EVT VT = N->getValueType(0).getVectorElementType();
226  unsigned NumOpers = N->getNumOperands();
227  SDValue Chain = N->getOperand(0);
228  EVT ValueVTs[] = {VT, MVT::Other};
229  SDLoc dl(N);
230 
231  SmallVector<SDValue, 4> Opers(NumOpers);
232 
233  // The Chain is the first operand.
234  Opers[0] = Chain;
235 
236  // Now process the remaining operands.
237  for (unsigned i = 1; i < NumOpers; ++i) {
238  SDValue Oper = N->getOperand(i);
239  EVT OperVT = Oper.getValueType();
240 
241  if (OperVT.isVector()) {
242  if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
243  Oper = GetScalarizedVector(Oper);
244  else
245  Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
246  OperVT.getVectorElementType(), Oper,
247  DAG.getVectorIdxConstant(0, dl));
248  }
249 
250  Opers[i] = Oper;
251  }
252 
253  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
254  Opers, N->getFlags());
255 
256  // Legalize the chain result - switch anything that used the old chain to
257  // use the new one.
258  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
259  return Result;
260 }
261 
262 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
263  unsigned ResNo) {
264  SDLoc DL(N);
265  EVT ResVT = N->getValueType(0);
266  EVT OvVT = N->getValueType(1);
267 
268  SDValue ScalarLHS, ScalarRHS;
269  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
270  ScalarLHS = GetScalarizedVector(N->getOperand(0));
271  ScalarRHS = GetScalarizedVector(N->getOperand(1));
272  } else {
273  SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
274  DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
275  DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
276  ScalarLHS = ElemsLHS[0];
277  ScalarRHS = ElemsRHS[0];
278  }
279 
280  SDVTList ScalarVTs = DAG.getVTList(
282  SDNode *ScalarNode = DAG.getNode(
283  N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
284  ScalarNode->setFlags(N->getFlags());
285 
286  // Replace the other vector result not being explicitly scalarized here.
287  unsigned OtherNo = 1 - ResNo;
288  EVT OtherVT = N->getValueType(OtherNo);
289  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
290  SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
291  } else {
292  SDValue OtherVal = DAG.getNode(
293  ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
294  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
295  }
296 
297  return SDValue(ScalarNode, ResNo);
298 }
299 
300 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
301  unsigned ResNo) {
302  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
303  return GetScalarizedVector(Op);
304 }
305 
306 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
307  SDValue Op = N->getOperand(0);
308  if (Op.getValueType().isVector()
309  && Op.getValueType().getVectorNumElements() == 1
310  && !isSimpleLegalType(Op.getValueType()))
311  Op = GetScalarizedVector(Op);
312  EVT NewVT = N->getValueType(0).getVectorElementType();
313  return DAG.getNode(ISD::BITCAST, SDLoc(N),
314  NewVT, Op);
315 }
316 
317 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
318  EVT EltVT = N->getValueType(0).getVectorElementType();
319  SDValue InOp = N->getOperand(0);
320  // The BUILD_VECTOR operands may be of wider element types and
321  // we may need to truncate them back to the requested return type.
322  if (EltVT.isInteger())
323  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
324  return InOp;
325 }
326 
327 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
328  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
329  N->getValueType(0).getVectorElementType(),
330  N->getOperand(0), N->getOperand(1));
331 }
332 
333 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
334  SDLoc DL(N);
335  SDValue Op = N->getOperand(0);
336  EVT OpVT = Op.getValueType();
337  // The result needs scalarizing, but it's not a given that the source does.
338  // See similar logic in ScalarizeVecRes_UnaryOp.
339  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
340  Op = GetScalarizedVector(Op);
341  } else {
342  EVT VT = OpVT.getVectorElementType();
343  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
344  DAG.getVectorIdxConstant(0, DL));
345  }
346  return DAG.getNode(ISD::FP_ROUND, DL,
347  N->getValueType(0).getVectorElementType(), Op,
348  N->getOperand(1));
349 }
350 
351 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
352  SDValue Op = GetScalarizedVector(N->getOperand(0));
353  return DAG.getNode(ISD::FPOWI, SDLoc(N),
354  Op.getValueType(), Op, N->getOperand(1));
355 }
356 
357 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
358  // The value to insert may have a wider type than the vector element type,
359  // so be sure to truncate it to the element type if necessary.
360  SDValue Op = N->getOperand(1);
361  EVT EltVT = N->getValueType(0).getVectorElementType();
362  if (Op.getValueType() != EltVT)
363  // FIXME: Can this happen for floating point types?
364  Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
365  return Op;
366 }
367 
368 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
369  assert(N->isUnindexed() && "Indexed vector load?");
370 
371  SDValue Result = DAG.getLoad(
372  ISD::UNINDEXED, N->getExtensionType(),
373  N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
374  N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
375  N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
376  N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
377 
378  // Legalize the chain result - switch anything that used the old chain to
379  // use the new one.
380  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
381  return Result;
382 }
383 
384 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
385  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
386  EVT DestVT = N->getValueType(0).getVectorElementType();
387  SDValue Op = N->getOperand(0);
388  EVT OpVT = Op.getValueType();
389  SDLoc DL(N);
390  // The result needs scalarizing, but it's not a given that the source does.
391  // This is a workaround for targets where it's impossible to scalarize the
392  // result of a conversion, because the source type is legal.
393  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
394  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
395  // legal and was not scalarized.
396  // See the similar logic in ScalarizeVecRes_SETCC
397  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
398  Op = GetScalarizedVector(Op);
399  } else {
400  EVT VT = OpVT.getVectorElementType();
401  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
402  DAG.getVectorIdxConstant(0, DL));
403  }
404  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
405 }
406 
407 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
408  EVT EltVT = N->getValueType(0).getVectorElementType();
409  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
410  SDValue LHS = GetScalarizedVector(N->getOperand(0));
411  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
412  LHS, DAG.getValueType(ExtVT));
413 }
414 
415 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
416  SDLoc DL(N);
417  SDValue Op = N->getOperand(0);
418 
419  EVT OpVT = Op.getValueType();
420  EVT OpEltVT = OpVT.getVectorElementType();
421  EVT EltVT = N->getValueType(0).getVectorElementType();
422 
423  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
424  Op = GetScalarizedVector(Op);
425  } else {
426  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
427  DAG.getVectorIdxConstant(0, DL));
428  }
429 
430  switch (N->getOpcode()) {
432  return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
434  return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
436  return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
437  }
438 
439  llvm_unreachable("Illegal extend_vector_inreg opcode");
440 }
441 
442 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
443  // If the operand is wider than the vector element type then it is implicitly
444  // truncated. Make that explicit here.
445  EVT EltVT = N->getValueType(0).getVectorElementType();
446  SDValue InOp = N->getOperand(0);
447  if (InOp.getValueType() != EltVT)
448  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
449  return InOp;
450 }
451 
452 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
453  SDValue Cond = N->getOperand(0);
454  EVT OpVT = Cond.getValueType();
455  SDLoc DL(N);
456  // The vselect result and true/value operands needs scalarizing, but it's
457  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
458  // See the similar logic in ScalarizeVecRes_SETCC
459  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
460  Cond = GetScalarizedVector(Cond);
461  } else {
462  EVT VT = OpVT.getVectorElementType();
464  DAG.getVectorIdxConstant(0, DL));
465  }
466 
467  SDValue LHS = GetScalarizedVector(N->getOperand(1));
468  TargetLowering::BooleanContent ScalarBool =
469  TLI.getBooleanContents(false, false);
470  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
471 
472  // If integer and float booleans have different contents then we can't
473  // reliably optimize in all cases. There is a full explanation for this in
474  // DAGCombiner::visitSELECT() where the same issue affects folding
475  // (select C, 0, 1) to (xor C, 1).
476  if (TLI.getBooleanContents(false, false) !=
477  TLI.getBooleanContents(false, true)) {
478  // At least try the common case where the boolean is generated by a
479  // comparison.
480  if (Cond->getOpcode() == ISD::SETCC) {
481  EVT OpVT = Cond->getOperand(0).getValueType();
482  ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
483  VecBool = TLI.getBooleanContents(OpVT);
484  } else
486  }
487 
488  EVT CondVT = Cond.getValueType();
489  if (ScalarBool != VecBool) {
490  switch (ScalarBool) {
492  break;
496  // Vector read from all ones, scalar expects a single 1 so mask.
497  Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
498  Cond, DAG.getConstant(1, SDLoc(N), CondVT));
499  break;
503  // Vector reads from a one, scalar from all ones so sign extend.
504  Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
505  Cond, DAG.getValueType(MVT::i1));
506  break;
507  }
508  }
509 
510  // Truncate the condition if needed
511  auto BoolVT = getSetCCResultType(CondVT);
512  if (BoolVT.bitsLT(CondVT))
513  Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
514 
515  return DAG.getSelect(SDLoc(N),
516  LHS.getValueType(), Cond, LHS,
517  GetScalarizedVector(N->getOperand(2)));
518 }
519 
520 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
521  SDValue LHS = GetScalarizedVector(N->getOperand(1));
522  return DAG.getSelect(SDLoc(N),
523  LHS.getValueType(), N->getOperand(0), LHS,
524  GetScalarizedVector(N->getOperand(2)));
525 }
526 
527 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
528  SDValue LHS = GetScalarizedVector(N->getOperand(2));
529  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
530  N->getOperand(0), N->getOperand(1),
531  LHS, GetScalarizedVector(N->getOperand(3)),
532  N->getOperand(4));
533 }
534 
535 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
536  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
537 }
538 
539 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
540  // Figure out if the scalar is the LHS or RHS and return it.
541  SDValue Arg = N->getOperand(2).getOperand(0);
542  if (Arg.isUndef())
543  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
544  unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
545  return GetScalarizedVector(N->getOperand(Op));
546 }
547 
548 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
549  SDValue Src = N->getOperand(0);
550  EVT SrcVT = Src.getValueType();
551  SDLoc dl(N);
552 
553  // Handle case where result is scalarized but operand is not
554  if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
555  Src = GetScalarizedVector(Src);
556  else
557  Src = DAG.getNode(
559  DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
560 
561  EVT DstVT = N->getValueType(0).getVectorElementType();
562  return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
563 }
564 
565 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
566  assert(N->getValueType(0).isVector() &&
567  N->getOperand(0).getValueType().isVector() &&
568  "Operand types must be vectors");
569  SDValue LHS = N->getOperand(0);
570  SDValue RHS = N->getOperand(1);
571  EVT OpVT = LHS.getValueType();
572  EVT NVT = N->getValueType(0).getVectorElementType();
573  SDLoc DL(N);
574 
575  // The result needs scalarizing, but it's not a given that the source does.
576  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
577  LHS = GetScalarizedVector(LHS);
578  RHS = GetScalarizedVector(RHS);
579  } else {
580  EVT VT = OpVT.getVectorElementType();
582  DAG.getVectorIdxConstant(0, DL));
584  DAG.getVectorIdxConstant(0, DL));
585  }
586 
587  // Turn it into a scalar SETCC.
588  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
589  N->getOperand(2));
590  // Vectors may have a different boolean contents to scalars. Promote the
591  // value appropriately.
592  ISD::NodeType ExtendCode =
594  return DAG.getNode(ExtendCode, DL, NVT, Res);
595 }
596 
597 SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
598  SDLoc DL(N);
599  SDValue Arg = N->getOperand(0);
600  SDValue Test = N->getOperand(1);
601  EVT ArgVT = Arg.getValueType();
602  EVT ResultVT = N->getValueType(0).getVectorElementType();
603 
604  if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
605  Arg = GetScalarizedVector(Arg);
606  } else {
607  EVT VT = ArgVT.getVectorElementType();
609  DAG.getVectorIdxConstant(0, DL));
610  }
611 
612  SDValue Res =
613  DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
614  // Vectors may have a different boolean contents to scalars. Promote the
615  // value appropriately.
616  ISD::NodeType ExtendCode =
618  return DAG.getNode(ExtendCode, DL, ResultVT, Res);
619 }
620 
621 //===----------------------------------------------------------------------===//
622 // Operand Vector Scalarization <1 x ty> -> ty.
623 //===----------------------------------------------------------------------===//
624 
625 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
626  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
627  dbgs() << "\n");
628  SDValue Res = SDValue();
629 
630  switch (N->getOpcode()) {
631  default:
632 #ifndef NDEBUG
633  dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
634  N->dump(&DAG);
635  dbgs() << "\n";
636 #endif
637  report_fatal_error("Do not know how to scalarize this operator's "
638  "operand!\n");
639  case ISD::BITCAST:
640  Res = ScalarizeVecOp_BITCAST(N);
641  break;
642  case ISD::ANY_EXTEND:
643  case ISD::ZERO_EXTEND:
644  case ISD::SIGN_EXTEND:
645  case ISD::TRUNCATE:
646  case ISD::FP_TO_SINT:
647  case ISD::FP_TO_UINT:
648  case ISD::SINT_TO_FP:
649  case ISD::UINT_TO_FP:
650  Res = ScalarizeVecOp_UnaryOp(N);
651  break;
656  Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
657  break;
658  case ISD::CONCAT_VECTORS:
659  Res = ScalarizeVecOp_CONCAT_VECTORS(N);
660  break;
662  Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
663  break;
664  case ISD::VSELECT:
665  Res = ScalarizeVecOp_VSELECT(N);
666  break;
667  case ISD::SETCC:
668  Res = ScalarizeVecOp_VSETCC(N);
669  break;
670  case ISD::STORE:
671  Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
672  break;
674  Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
675  break;
676  case ISD::FP_ROUND:
677  Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
678  break;
680  Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
681  break;
682  case ISD::FP_EXTEND:
683  Res = ScalarizeVecOp_FP_EXTEND(N);
684  break;
685  case ISD::VECREDUCE_FADD:
686  case ISD::VECREDUCE_FMUL:
687  case ISD::VECREDUCE_ADD:
688  case ISD::VECREDUCE_MUL:
689  case ISD::VECREDUCE_AND:
690  case ISD::VECREDUCE_OR:
691  case ISD::VECREDUCE_XOR:
692  case ISD::VECREDUCE_SMAX:
693  case ISD::VECREDUCE_SMIN:
694  case ISD::VECREDUCE_UMAX:
695  case ISD::VECREDUCE_UMIN:
696  case ISD::VECREDUCE_FMAX:
697  case ISD::VECREDUCE_FMIN:
698  Res = ScalarizeVecOp_VECREDUCE(N);
699  break;
702  Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
703  break;
704  }
705 
706  // If the result is null, the sub-method took care of registering results etc.
707  if (!Res.getNode()) return false;
708 
709  // If the result is N, the sub-method updated N in place. Tell the legalizer
710  // core about this.
711  if (Res.getNode() == N)
712  return true;
713 
714  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
715  "Invalid operand expansion");
716 
717  ReplaceValueWith(SDValue(N, 0), Res);
718  return false;
719 }
720 
721 /// If the value to convert is a vector that needs to be scalarized, it must be
722 /// <1 x ty>. Convert the element instead.
723 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
724  SDValue Elt = GetScalarizedVector(N->getOperand(0));
725  return DAG.getNode(ISD::BITCAST, SDLoc(N),
726  N->getValueType(0), Elt);
727 }
728 
729 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
730 /// Do the operation on the element instead.
731 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
732  assert(N->getValueType(0).getVectorNumElements() == 1 &&
733  "Unexpected vector type!");
734  SDValue Elt = GetScalarizedVector(N->getOperand(0));
735  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
736  N->getValueType(0).getScalarType(), Elt);
737  // Revectorize the result so the types line up with what the uses of this
738  // expression expect.
739  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
740 }
741 
742 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
743 /// Do the strict FP operation on the element instead.
744 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
745  assert(N->getValueType(0).getVectorNumElements() == 1 &&
746  "Unexpected vector type!");
747  SDValue Elt = GetScalarizedVector(N->getOperand(1));
748  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
749  { N->getValueType(0).getScalarType(), MVT::Other },
750  { N->getOperand(0), Elt });
751  // Legalize the chain result - switch anything that used the old chain to
752  // use the new one.
753  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
754  // Revectorize the result so the types line up with what the uses of this
755  // expression expect.
756  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
757 
758  // Do our own replacement and return SDValue() to tell the caller that we
759  // handled all replacements since caller can only handle a single result.
760  ReplaceValueWith(SDValue(N, 0), Res);
761  return SDValue();
762 }
763 
764 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
765 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
766  SmallVector<SDValue, 8> Ops(N->getNumOperands());
767  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
768  Ops[i] = GetScalarizedVector(N->getOperand(i));
769  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
770 }
771 
772 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
773 /// so just return the element, ignoring the index.
774 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
775  EVT VT = N->getValueType(0);
776  SDValue Res = GetScalarizedVector(N->getOperand(0));
777  if (Res.getValueType() != VT)
778  Res = VT.isFloatingPoint()
779  ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
780  : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
781  return Res;
782 }
783 
784 /// If the input condition is a vector that needs to be scalarized, it must be
785 /// <1 x i1>, so just convert to a normal ISD::SELECT
786 /// (still with vector output type since that was acceptable if we got here).
787 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
788  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
789  EVT VT = N->getValueType(0);
790 
791  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
792  N->getOperand(2));
793 }
794 
795 /// If the operand is a vector that needs to be scalarized then the
796 /// result must be v1i1, so just convert to a scalar SETCC and wrap
797 /// with a scalar_to_vector since the res type is legal if we got here
798 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
799  assert(N->getValueType(0).isVector() &&
800  N->getOperand(0).getValueType().isVector() &&
801  "Operand types must be vectors");
802  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
803 
804  EVT VT = N->getValueType(0);
805  SDValue LHS = GetScalarizedVector(N->getOperand(0));
806  SDValue RHS = GetScalarizedVector(N->getOperand(1));
807 
808  EVT OpVT = N->getOperand(0).getValueType();
809  EVT NVT = VT.getVectorElementType();
810  SDLoc DL(N);
811  // Turn it into a scalar SETCC.
812  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
813  N->getOperand(2));
814 
815  // Vectors may have a different boolean contents to scalars. Promote the
816  // value appropriately.
817  ISD::NodeType ExtendCode =
819 
820  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
821 
822  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
823 }
824 
825 /// If the value to store is a vector that needs to be scalarized, it must be
826 /// <1 x ty>. Just store the element.
827 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
828  assert(N->isUnindexed() && "Indexed store of one-element vector?");
829  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
830  SDLoc dl(N);
831 
832  if (N->isTruncatingStore())
833  return DAG.getTruncStore(
834  N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
835  N->getBasePtr(), N->getPointerInfo(),
836  N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
837  N->getMemOperand()->getFlags(), N->getAAInfo());
838 
839  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
840  N->getBasePtr(), N->getPointerInfo(),
841  N->getOriginalAlign(), N->getMemOperand()->getFlags(),
842  N->getAAInfo());
843 }
844 
845 /// If the value to round is a vector that needs to be scalarized, it must be
846 /// <1 x ty>. Convert the element instead.
847 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
848  assert(OpNo == 0 && "Wrong operand for scalarization!");
849  SDValue Elt = GetScalarizedVector(N->getOperand(0));
850  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
851  N->getValueType(0).getVectorElementType(), Elt,
852  N->getOperand(1));
853  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
854 }
855 
856 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
857  unsigned OpNo) {
858  assert(OpNo == 1 && "Wrong operand for scalarization!");
859  SDValue Elt = GetScalarizedVector(N->getOperand(1));
861  { N->getValueType(0).getVectorElementType(),
862  MVT::Other },
863  { N->getOperand(0), Elt, N->getOperand(2) });
864  // Legalize the chain result - switch anything that used the old chain to
865  // use the new one.
866  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
867 
868  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
869 
870  // Do our own replacement and return SDValue() to tell the caller that we
871  // handled all replacements since caller can only handle a single result.
872  ReplaceValueWith(SDValue(N, 0), Res);
873  return SDValue();
874 }
875 
876 /// If the value to extend is a vector that needs to be scalarized, it must be
877 /// <1 x ty>. Convert the element instead.
878 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
879  SDValue Elt = GetScalarizedVector(N->getOperand(0));
880  SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
881  N->getValueType(0).getVectorElementType(), Elt);
882  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
883 }
884 
885 /// If the value to extend is a vector that needs to be scalarized, it must be
886 /// <1 x ty>. Convert the element instead.
887 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
888  SDValue Elt = GetScalarizedVector(N->getOperand(1));
889  SDValue Res =
891  {N->getValueType(0).getVectorElementType(), MVT::Other},
892  {N->getOperand(0), Elt});
893  // Legalize the chain result - switch anything that used the old chain to
894  // use the new one.
895  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
896 
897  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
898 
899  // Do our own replacement and return SDValue() to tell the caller that we
900  // handled all replacements since caller can only handle a single result.
901  ReplaceValueWith(SDValue(N, 0), Res);
902  return SDValue();
903 }
904 
905 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
906  SDValue Res = GetScalarizedVector(N->getOperand(0));
907  // Result type may be wider than element type.
908  if (Res.getValueType() != N->getValueType(0))
909  Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
910  return Res;
911 }
912 
913 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
914  SDValue AccOp = N->getOperand(0);
915  SDValue VecOp = N->getOperand(1);
916 
917  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
918 
919  SDValue Op = GetScalarizedVector(VecOp);
920  return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
921  AccOp, Op, N->getFlags());
922 }
923 
924 //===----------------------------------------------------------------------===//
925 // Result Vector Splitting
926 //===----------------------------------------------------------------------===//
927 
928 /// This method is called when the specified result of the specified node is
929 /// found to need vector splitting. At this point, the node may also have
930 /// invalid operands or may have other results that need legalization, we just
931 /// know that (at least) one result needs vector splitting.
932 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
933  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
934  SDValue Lo, Hi;
935 
936  // See if the target wants to custom expand this node.
937  if (CustomLowerNode(N, N->getValueType(ResNo), true))
938  return;
939 
940  switch (N->getOpcode()) {
941  default:
942 #ifndef NDEBUG
943  dbgs() << "SplitVectorResult #" << ResNo << ": ";
944  N->dump(&DAG);
945  dbgs() << "\n";
946 #endif
947  report_fatal_error("Do not know how to split the result of this "
948  "operator!\n");
949 
950  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
951  case ISD::VSELECT:
952  case ISD::SELECT:
953  case ISD::VP_MERGE:
954  case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
955  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
956  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
957  case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
958  case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
959  case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
960  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
961  case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
962  case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
963  case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
964  case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
965  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
966  case ISD::SPLAT_VECTOR:
968  SplitVecRes_ScalarOp(N, Lo, Hi);
969  break;
970  case ISD::STEP_VECTOR:
971  SplitVecRes_STEP_VECTOR(N, Lo, Hi);
972  break;
973  case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
974  case ISD::LOAD:
975  SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
976  break;
977  case ISD::VP_LOAD:
978  SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
979  break;
980  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
981  SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi);
982  break;
983  case ISD::MLOAD:
984  SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
985  break;
986  case ISD::MGATHER:
987  case ISD::VP_GATHER:
988  SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
989  break;
990  case ISD::SETCC:
991  case ISD::VP_SETCC:
992  SplitVecRes_SETCC(N, Lo, Hi);
993  break;
994  case ISD::VECTOR_REVERSE:
995  SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
996  break;
997  case ISD::VECTOR_SHUFFLE:
998  SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
999  break;
1000  case ISD::VECTOR_SPLICE:
1001  SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1002  break;
1003  case ISD::VAARG:
1004  SplitVecRes_VAARG(N, Lo, Hi);
1005  break;
1006 
1010  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1011  break;
1012 
1013  case ISD::ABS:
1014  case ISD::BITREVERSE:
1015  case ISD::BSWAP:
1016  case ISD::VP_BSWAP:
1017  case ISD::CTLZ:
1018  case ISD::CTTZ:
1019  case ISD::CTLZ_ZERO_UNDEF:
1020  case ISD::CTTZ_ZERO_UNDEF:
1021  case ISD::CTPOP:
1022  case ISD::FABS: case ISD::VP_FABS:
1023  case ISD::FCEIL:
1024  case ISD::VP_FCEIL:
1025  case ISD::FCOS:
1026  case ISD::FEXP:
1027  case ISD::FEXP2:
1028  case ISD::FFLOOR:
1029  case ISD::VP_FFLOOR:
1030  case ISD::FLOG:
1031  case ISD::FLOG10:
1032  case ISD::FLOG2:
1033  case ISD::FNEARBYINT:
1034  case ISD::VP_FNEARBYINT:
1035  case ISD::FNEG: case ISD::VP_FNEG:
1036  case ISD::FREEZE:
1037  case ISD::ARITH_FENCE:
1038  case ISD::FP_EXTEND:
1039  case ISD::VP_FP_EXTEND:
1040  case ISD::FP_ROUND:
1041  case ISD::VP_FP_ROUND:
1042  case ISD::FP_TO_SINT:
1043  case ISD::VP_FP_TO_SINT:
1044  case ISD::FP_TO_UINT:
1045  case ISD::VP_FP_TO_UINT:
1046  case ISD::FRINT:
1047  case ISD::VP_FRINT:
1048  case ISD::FROUND:
1049  case ISD::VP_FROUND:
1050  case ISD::FROUNDEVEN:
1051  case ISD::VP_FROUNDEVEN:
1052  case ISD::FSIN:
1053  case ISD::FSQRT: case ISD::VP_SQRT:
1054  case ISD::FTRUNC:
1055  case ISD::VP_FROUNDTOZERO:
1056  case ISD::SINT_TO_FP:
1057  case ISD::VP_SINT_TO_FP:
1058  case ISD::TRUNCATE:
1059  case ISD::VP_TRUNCATE:
1060  case ISD::UINT_TO_FP:
1061  case ISD::VP_UINT_TO_FP:
1062  case ISD::FCANONICALIZE:
1063  SplitVecRes_UnaryOp(N, Lo, Hi);
1064  break;
1065 
1066  case ISD::ANY_EXTEND:
1067  case ISD::SIGN_EXTEND:
1068  case ISD::ZERO_EXTEND:
1069  case ISD::VP_SIGN_EXTEND:
1070  case ISD::VP_ZERO_EXTEND:
1071  SplitVecRes_ExtendOp(N, Lo, Hi);
1072  break;
1073 
1074  case ISD::ADD: case ISD::VP_ADD:
1075  case ISD::SUB: case ISD::VP_SUB:
1076  case ISD::MUL: case ISD::VP_MUL:
1077  case ISD::MULHS:
1078  case ISD::MULHU:
1079  case ISD::FADD: case ISD::VP_FADD:
1080  case ISD::FSUB: case ISD::VP_FSUB:
1081  case ISD::FMUL: case ISD::VP_FMUL:
1082  case ISD::FMINNUM: case ISD::VP_FMINNUM:
1083  case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1084  case ISD::FMINIMUM:
1085  case ISD::FMAXIMUM:
1086  case ISD::SDIV: case ISD::VP_SDIV:
1087  case ISD::UDIV: case ISD::VP_UDIV:
1088  case ISD::FDIV: case ISD::VP_FDIV:
1089  case ISD::FPOW:
1090  case ISD::AND: case ISD::VP_AND:
1091  case ISD::OR: case ISD::VP_OR:
1092  case ISD::XOR: case ISD::VP_XOR:
1093  case ISD::SHL: case ISD::VP_SHL:
1094  case ISD::SRA: case ISD::VP_ASHR:
1095  case ISD::SRL: case ISD::VP_LSHR:
1096  case ISD::UREM: case ISD::VP_UREM:
1097  case ISD::SREM: case ISD::VP_SREM:
1098  case ISD::FREM: case ISD::VP_FREM:
1099  case ISD::SMIN: case ISD::VP_SMIN:
1100  case ISD::SMAX: case ISD::VP_SMAX:
1101  case ISD::UMIN: case ISD::VP_UMIN:
1102  case ISD::UMAX: case ISD::VP_UMAX:
1103  case ISD::SADDSAT:
1104  case ISD::UADDSAT:
1105  case ISD::SSUBSAT:
1106  case ISD::USUBSAT:
1107  case ISD::SSHLSAT:
1108  case ISD::USHLSAT:
1109  case ISD::ROTL:
1110  case ISD::ROTR:
1111  case ISD::VP_FCOPYSIGN:
1112  SplitVecRes_BinOp(N, Lo, Hi);
1113  break;
1114  case ISD::FMA: case ISD::VP_FMA:
1115  case ISD::FSHL:
1116  case ISD::VP_FSHL:
1117  case ISD::FSHR:
1118  case ISD::VP_FSHR:
1119  SplitVecRes_TernaryOp(N, Lo, Hi);
1120  break;
1121 
1122 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1123  case ISD::STRICT_##DAGN:
1124 #include "llvm/IR/ConstrainedOps.def"
1125  SplitVecRes_StrictFPOp(N, Lo, Hi);
1126  break;
1127 
1128  case ISD::FP_TO_UINT_SAT:
1129  case ISD::FP_TO_SINT_SAT:
1130  SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1131  break;
1132 
1133  case ISD::UADDO:
1134  case ISD::SADDO:
1135  case ISD::USUBO:
1136  case ISD::SSUBO:
1137  case ISD::UMULO:
1138  case ISD::SMULO:
1139  SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1140  break;
1141  case ISD::SMULFIX:
1142  case ISD::SMULFIXSAT:
1143  case ISD::UMULFIX:
1144  case ISD::UMULFIXSAT:
1145  case ISD::SDIVFIX:
1146  case ISD::SDIVFIXSAT:
1147  case ISD::UDIVFIX:
1148  case ISD::UDIVFIXSAT:
1149  SplitVecRes_FIX(N, Lo, Hi);
1150  break;
1151  }
1152 
1153  // If Lo/Hi is null, the sub-method took care of registering results etc.
1154  if (Lo.getNode())
1155  SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1156 }
1157 
1158 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1160  uint64_t *ScaledOffset) {
1161  SDLoc DL(N);
1162  unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1163 
1164  if (MemVT.isScalableVector()) {
1166  SDValue BytesIncrement = DAG.getVScale(
1167  DL, Ptr.getValueType(),
1168  APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1169  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1170  Flags.setNoUnsignedWrap(true);
1171  if (ScaledOffset)
1172  *ScaledOffset += IncrementSize;
1173  Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1174  Flags);
1175  } else {
1176  MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1177  // Increment the pointer to the other half.
1178  Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1179  }
1180 }
1181 
1182 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1183  return SplitMask(Mask, SDLoc(Mask));
1184 }
1185 
1186 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1187  const SDLoc &DL) {
1188  SDValue MaskLo, MaskHi;
1189  EVT MaskVT = Mask.getValueType();
1190  if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1191  GetSplitVector(Mask, MaskLo, MaskHi);
1192  else
1193  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1194  return std::make_pair(MaskLo, MaskHi);
1195 }
1196 
1197 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1198  SDValue LHSLo, LHSHi;
1199  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1200  SDValue RHSLo, RHSHi;
1201  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1202  SDLoc dl(N);
1203 
1204  const SDNodeFlags Flags = N->getFlags();
1205  unsigned Opcode = N->getOpcode();
1206  if (N->getNumOperands() == 2) {
1207  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1208  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1209  return;
1210  }
1211 
1212  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1213  assert(N->isVPOpcode() && "Expected VP opcode");
1214 
1215  SDValue MaskLo, MaskHi;
1216  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1217 
1218  SDValue EVLLo, EVLHi;
1219  std::tie(EVLLo, EVLHi) =
1220  DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1221 
1222  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1223  {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1224  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1225  {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1226 }
1227 
1228 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1229  SDValue &Hi) {
1230  SDValue Op0Lo, Op0Hi;
1231  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1232  SDValue Op1Lo, Op1Hi;
1233  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1234  SDValue Op2Lo, Op2Hi;
1235  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1236  SDLoc dl(N);
1237 
1238  const SDNodeFlags Flags = N->getFlags();
1239  unsigned Opcode = N->getOpcode();
1240  if (N->getNumOperands() == 3) {
1241  Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1242  Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1243  return;
1244  }
1245 
1246  assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1247  assert(N->isVPOpcode() && "Expected VP opcode");
1248 
1249  SDValue MaskLo, MaskHi;
1250  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1251 
1252  SDValue EVLLo, EVLHi;
1253  std::tie(EVLLo, EVLHi) =
1254  DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1255 
1256  Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1257  {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1258  Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1259  {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1260 }
1261 
1262 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1263  SDValue LHSLo, LHSHi;
1264  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1265  SDValue RHSLo, RHSHi;
1266  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1267  SDLoc dl(N);
1268  SDValue Op2 = N->getOperand(2);
1269 
1270  unsigned Opcode = N->getOpcode();
1271  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1272  N->getFlags());
1273  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1274  N->getFlags());
1275 }
1276 
1277 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1278  SDValue &Hi) {
1279  // We know the result is a vector. The input may be either a vector or a
1280  // scalar value.
1281  EVT LoVT, HiVT;
1282  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1283  SDLoc dl(N);
1284 
1285  SDValue InOp = N->getOperand(0);
1286  EVT InVT = InOp.getValueType();
1287 
1288  // Handle some special cases efficiently.
1289  switch (getTypeAction(InVT)) {
1297  break;
1300  // A scalar to vector conversion, where the scalar needs expansion.
1301  // If the vector is being split in two then we can just convert the
1302  // expanded pieces.
1303  if (LoVT == HiVT) {
1304  GetExpandedOp(InOp, Lo, Hi);
1305  if (DAG.getDataLayout().isBigEndian())
1306  std::swap(Lo, Hi);
1307  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1308  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1309  return;
1310  }
1311  break;
1313  // If the input is a vector that needs to be split, convert each split
1314  // piece of the input now.
1315  GetSplitVector(InOp, Lo, Hi);
1316  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1317  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1318  return;
1320  report_fatal_error("Scalarization of scalable vectors is not supported.");
1321  }
1322 
1323  // In the general case, convert the input to an integer and split it by hand.
1324  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1325  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1326  if (DAG.getDataLayout().isBigEndian())
1327  std::swap(LoIntVT, HiIntVT);
1328 
1329  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1330 
1331  if (DAG.getDataLayout().isBigEndian())
1332  std::swap(Lo, Hi);
1333  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1334  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1335 }
1336 
1337 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1338  SDValue &Hi) {
1339  EVT LoVT, HiVT;
1340  SDLoc dl(N);
1341  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1342  unsigned LoNumElts = LoVT.getVectorNumElements();
1343  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1344  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1345 
1346  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1347  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1348 }
1349 
1350 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1351  SDValue &Hi) {
1352  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1353  SDLoc dl(N);
1354  unsigned NumSubvectors = N->getNumOperands() / 2;
1355  if (NumSubvectors == 1) {
1356  Lo = N->getOperand(0);
1357  Hi = N->getOperand(1);
1358  return;
1359  }
1360 
1361  EVT LoVT, HiVT;
1362  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1363 
1364  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1365  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1366 
1367  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1368  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1369 }
1370 
1371 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1372  SDValue &Hi) {
1373  SDValue Vec = N->getOperand(0);
1374  SDValue Idx = N->getOperand(1);
1375  SDLoc dl(N);
1376 
1377  EVT LoVT, HiVT;
1378  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1379 
1380  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1381  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1382  Hi = DAG.getNode(
1383  ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1384  DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1385 }
1386 
1387 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1388  SDValue &Hi) {
1389  SDValue Vec = N->getOperand(0);
1390  SDValue SubVec = N->getOperand(1);
1391  SDValue Idx = N->getOperand(2);
1392  SDLoc dl(N);
1393  GetSplitVector(Vec, Lo, Hi);
1394 
1395  EVT VecVT = Vec.getValueType();
1396  EVT LoVT = Lo.getValueType();
1397  EVT SubVecVT = SubVec.getValueType();
1398  unsigned VecElems = VecVT.getVectorMinNumElements();
1399  unsigned SubElems = SubVecVT.getVectorMinNumElements();
1400  unsigned LoElems = LoVT.getVectorMinNumElements();
1401 
1402  // If we know the index is in the first half, and we know the subvector
1403  // doesn't cross the boundary between the halves, we can avoid spilling the
1404  // vector, and insert into the lower half of the split vector directly.
1405  unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1406  if (IdxVal + SubElems <= LoElems) {
1407  Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1408  return;
1409  }
1410  // Similarly if the subvector is fully in the high half, but mind that we
1411  // can't tell whether a fixed-length subvector is fully within the high half
1412  // of a scalable vector.
1413  if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1414  IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1415  Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1416  DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1417  return;
1418  }
1419 
1420  // Spill the vector to the stack.
1421  // In cases where the vector is illegal it will be broken down into parts
1422  // and stored in parts - we should use the alignment for the smallest part.
1423  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1424  SDValue StackPtr =
1425  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1426  auto &MF = DAG.getMachineFunction();
1427  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1428  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1429 
1430  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1431  SmallestAlign);
1432 
1433  // Store the new subvector into the specified index.
1434  SDValue SubVecPtr =
1435  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1436  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1438 
1439  // Load the Lo part from the stack slot.
1440  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1441  SmallestAlign);
1442 
1443  // Increment the pointer to the other part.
1444  auto *Load = cast<LoadSDNode>(Lo);
1445  MachinePointerInfo MPI = Load->getPointerInfo();
1446  IncrementPointer(Load, LoVT, MPI, StackPtr);
1447 
1448  // Load the Hi part from the stack slot.
1449  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1450 }
1451 
1452 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1453  SDValue &Hi) {
1454  SDLoc dl(N);
1455  GetSplitVector(N->getOperand(0), Lo, Hi);
1456  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1457  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1458 }
1459 
1460 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1461  SDValue &Hi) {
1462  SDValue LHSLo, LHSHi;
1463  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1464  SDLoc DL(N);
1465 
1466  SDValue RHSLo, RHSHi;
1467  SDValue RHS = N->getOperand(1);
1468  EVT RHSVT = RHS.getValueType();
1469  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1470  GetSplitVector(RHS, RHSLo, RHSHi);
1471  else
1472  std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1473 
1474 
1475  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1476  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1477 }
1478 
1479 void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1480  SDValue &Hi) {
1481  SDLoc DL(N);
1482  SDValue ArgLo, ArgHi;
1483  SDValue Test = N->getOperand(1);
1484  SDValue FpValue = N->getOperand(0);
1485  if (getTypeAction(FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1486  GetSplitVector(FpValue, ArgLo, ArgHi);
1487  else
1488  std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue));
1489  EVT LoVT, HiVT;
1490  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1491 
1492  Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1493  Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1494 }
1495 
1496 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1497  SDValue &Hi) {
1498  SDValue LHSLo, LHSHi;
1499  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1500  SDLoc dl(N);
1501 
1502  EVT LoVT, HiVT;
1503  std::tie(LoVT, HiVT) =
1504  DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1505 
1506  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1507  DAG.getValueType(LoVT));
1508  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1509  DAG.getValueType(HiVT));
1510 }
1511 
1512 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1513  SDValue &Hi) {
1514  unsigned Opcode = N->getOpcode();
1515  SDValue N0 = N->getOperand(0);
1516 
1517  SDLoc dl(N);
1518  SDValue InLo, InHi;
1519 
1520  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1521  GetSplitVector(N0, InLo, InHi);
1522  else
1523  std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1524 
1525  EVT InLoVT = InLo.getValueType();
1526  unsigned InNumElements = InLoVT.getVectorNumElements();
1527 
1528  EVT OutLoVT, OutHiVT;
1529  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1530  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1531  assert((2 * OutNumElements) <= InNumElements &&
1532  "Illegal extend vector in reg split");
1533 
1534  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1535  // input vector (i.e. we only use InLo):
1536  // OutLo will extend the first OutNumElements from InLo.
1537  // OutHi will extend the next OutNumElements from InLo.
1538 
1539  // Shuffle the elements from InLo for OutHi into the bottom elements to
1540  // create a 'fake' InHi.
1541  SmallVector<int, 8> SplitHi(InNumElements, -1);
1542  for (unsigned i = 0; i != OutNumElements; ++i)
1543  SplitHi[i] = i + OutNumElements;
1544  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1545 
1546  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1547  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1548 }
1549 
1550 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1551  SDValue &Hi) {
1552  unsigned NumOps = N->getNumOperands();
1553  SDValue Chain = N->getOperand(0);
1554  EVT LoVT, HiVT;
1555  SDLoc dl(N);
1556  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1557 
1558  SmallVector<SDValue, 4> OpsLo(NumOps);
1559  SmallVector<SDValue, 4> OpsHi(NumOps);
1560 
1561  // The Chain is the first operand.
1562  OpsLo[0] = Chain;
1563  OpsHi[0] = Chain;
1564 
1565  // Now process the remaining operands.
1566  for (unsigned i = 1; i < NumOps; ++i) {
1567  SDValue Op = N->getOperand(i);
1568  SDValue OpLo = Op;
1569  SDValue OpHi = Op;
1570 
1571  EVT InVT = Op.getValueType();
1572  if (InVT.isVector()) {
1573  // If the input also splits, handle it directly for a
1574  // compile time speedup. Otherwise split it by hand.
1575  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1576  GetSplitVector(Op, OpLo, OpHi);
1577  else
1578  std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1579  }
1580 
1581  OpsLo[i] = OpLo;
1582  OpsHi[i] = OpHi;
1583  }
1584 
1585  EVT LoValueVTs[] = {LoVT, MVT::Other};
1586  EVT HiValueVTs[] = {HiVT, MVT::Other};
1587  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1588  N->getFlags());
1589  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1590  N->getFlags());
1591 
1592  // Build a factor node to remember that this Op is independent of the
1593  // other one.
1594  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1595  Lo.getValue(1), Hi.getValue(1));
1596 
1597  // Legalize the chain result - switch anything that used the old chain to
1598  // use the new one.
1599  ReplaceValueWith(SDValue(N, 1), Chain);
1600 }
1601 
1602 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1603  SDValue Chain = N->getOperand(0);
1604  EVT VT = N->getValueType(0);
1605  unsigned NE = VT.getVectorNumElements();
1606  EVT EltVT = VT.getVectorElementType();
1607  SDLoc dl(N);
1608 
1609  SmallVector<SDValue, 8> Scalars;
1610  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1611 
1612  // If ResNE is 0, fully unroll the vector op.
1613  if (ResNE == 0)
1614  ResNE = NE;
1615  else if (NE > ResNE)
1616  NE = ResNE;
1617 
1618  //The results of each unrolled operation, including the chain.
1619  EVT ChainVTs[] = {EltVT, MVT::Other};
1620  SmallVector<SDValue, 8> Chains;
1621 
1622  unsigned i;
1623  for (i = 0; i != NE; ++i) {
1624  Operands[0] = Chain;
1625  for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1626  SDValue Operand = N->getOperand(j);
1627  EVT OperandVT = Operand.getValueType();
1628  if (OperandVT.isVector()) {
1629  EVT OperandEltVT = OperandVT.getVectorElementType();
1630  Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1631  Operand, DAG.getVectorIdxConstant(i, dl));
1632  } else {
1633  Operands[j] = Operand;
1634  }
1635  }
1636  SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1637  Scalar.getNode()->setFlags(N->getFlags());
1638 
1639  //Add in the scalar as well as its chain value to the
1640  //result vectors.
1641  Scalars.push_back(Scalar);
1642  Chains.push_back(Scalar.getValue(1));
1643  }
1644 
1645  for (; i < ResNE; ++i)
1646  Scalars.push_back(DAG.getUNDEF(EltVT));
1647 
1648  // Build a new factor node to connect the chain back together.
1649  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1650  ReplaceValueWith(SDValue(N, 1), Chain);
1651 
1652  // Create a new BUILD_VECTOR node
1653  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1654  return DAG.getBuildVector(VecVT, dl, Scalars);
1655 }
1656 
1657 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1658  SDValue &Lo, SDValue &Hi) {
1659  SDLoc dl(N);
1660  EVT ResVT = N->getValueType(0);
1661  EVT OvVT = N->getValueType(1);
1662  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1663  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1664  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1665 
1666  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1667  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1668  GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1669  GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1670  } else {
1671  std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1672  std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1673  }
1674 
1675  unsigned Opcode = N->getOpcode();
1676  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1677  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1678  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1679  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1680  LoNode->setFlags(N->getFlags());
1681  HiNode->setFlags(N->getFlags());
1682 
1683  Lo = SDValue(LoNode, ResNo);
1684  Hi = SDValue(HiNode, ResNo);
1685 
1686  // Replace the other vector result not being explicitly split here.
1687  unsigned OtherNo = 1 - ResNo;
1688  EVT OtherVT = N->getValueType(OtherNo);
1689  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1690  SetSplitVector(SDValue(N, OtherNo),
1691  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1692  } else {
1693  SDValue OtherVal = DAG.getNode(
1694  ISD::CONCAT_VECTORS, dl, OtherVT,
1695  SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1696  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1697  }
1698 }
1699 
1700 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1701  SDValue &Hi) {
1702  SDValue Vec = N->getOperand(0);
1703  SDValue Elt = N->getOperand(1);
1704  SDValue Idx = N->getOperand(2);
1705  SDLoc dl(N);
1706  GetSplitVector(Vec, Lo, Hi);
1707 
1708  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1709  unsigned IdxVal = CIdx->getZExtValue();
1710  unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1711  if (IdxVal < LoNumElts) {
1712  Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1713  Lo.getValueType(), Lo, Elt, Idx);
1714  return;
1715  } else if (!Vec.getValueType().isScalableVector()) {
1716  Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1717  DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1718  return;
1719  }
1720  }
1721 
1722  // See if the target wants to custom expand this node.
1723  if (CustomLowerNode(N, N->getValueType(0), true))
1724  return;
1725 
1726  // Make the vector elements byte-addressable if they aren't already.
1727  EVT VecVT = Vec.getValueType();
1728  EVT EltVT = VecVT.getVectorElementType();
1729  if (VecVT.getScalarSizeInBits() < 8) {
1730  EltVT = MVT::i8;
1731  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1732  VecVT.getVectorElementCount());
1733  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1734  // Extend the element type to match if needed.
1735  if (EltVT.bitsGT(Elt.getValueType()))
1736  Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1737  }
1738 
1739  // Spill the vector to the stack.
1740  // In cases where the vector is illegal it will be broken down into parts
1741  // and stored in parts - we should use the alignment for the smallest part.
1742  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1743  SDValue StackPtr =
1744  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1745  auto &MF = DAG.getMachineFunction();
1746  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1747  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1748 
1749  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1750  SmallestAlign);
1751 
1752  // Store the new element. This may be larger than the vector element type,
1753  // so use a truncating store.
1754  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1755  Store = DAG.getTruncStore(
1756  Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1757  commonAlignment(SmallestAlign,
1758  EltVT.getFixedSizeInBits() / 8));
1759 
1760  EVT LoVT, HiVT;
1761  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1762 
1763  // Load the Lo part from the stack slot.
1764  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1765 
1766  // Increment the pointer to the other part.
1767  auto Load = cast<LoadSDNode>(Lo);
1768  MachinePointerInfo MPI = Load->getPointerInfo();
1769  IncrementPointer(Load, LoVT, MPI, StackPtr);
1770 
1771  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1772 
1773  // If we adjusted the original type, we need to truncate the results.
1774  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1775  if (LoVT != Lo.getValueType())
1776  Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1777  if (HiVT != Hi.getValueType())
1778  Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1779 }
1780 
1781 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1782  SDValue &Hi) {
1783  EVT LoVT, HiVT;
1784  SDLoc dl(N);
1785  assert(N->getValueType(0).isScalableVector() &&
1786  "Only scalable vectors are supported for STEP_VECTOR");
1787  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1788  SDValue Step = N->getOperand(0);
1789 
1790  Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1791 
1792  // Hi = Lo + (EltCnt * Step)
1793  EVT EltVT = Step.getValueType();
1794  APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1795  SDValue StartOfHi =
1796  DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1797  StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1798  StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1799 
1800  Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1801  Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1802 }
1803 
1804 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1805  SDValue &Hi) {
1806  EVT LoVT, HiVT;
1807  SDLoc dl(N);
1808  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1809  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1810  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1811  Hi = DAG.getUNDEF(HiVT);
1812  } else {
1813  assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1814  Hi = Lo;
1815  }
1816 }
1817 
1818 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1819  SDValue &Hi) {
1820  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1821  EVT LoVT, HiVT;
1822  SDLoc dl(LD);
1823  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1824 
1825  ISD::LoadExtType ExtType = LD->getExtensionType();
1826  SDValue Ch = LD->getChain();
1827  SDValue Ptr = LD->getBasePtr();
1828  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1829  EVT MemoryVT = LD->getMemoryVT();
1830  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1831  AAMDNodes AAInfo = LD->getAAInfo();
1832 
1833  EVT LoMemVT, HiMemVT;
1834  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1835 
1836  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1837  SDValue Value, NewChain;
1838  std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1839  std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1840  ReplaceValueWith(SDValue(LD, 1), NewChain);
1841  return;
1842  }
1843 
1844  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1845  LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1846  MMOFlags, AAInfo);
1847 
1848  MachinePointerInfo MPI;
1849  IncrementPointer(LD, LoMemVT, MPI, Ptr);
1850 
1851  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1852  HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1853 
1854  // Build a factor node to remember that this load is independent of the
1855  // other one.
1856  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1857  Hi.getValue(1));
1858 
1859  // Legalize the chain result - switch anything that used the old chain to
1860  // use the new one.
1861  ReplaceValueWith(SDValue(LD, 1), Ch);
1862 }
1863 
1864 void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1865  SDValue &Hi) {
1866  assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1867  EVT LoVT, HiVT;
1868  SDLoc dl(LD);
1869  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1870 
1871  ISD::LoadExtType ExtType = LD->getExtensionType();
1872  SDValue Ch = LD->getChain();
1873  SDValue Ptr = LD->getBasePtr();
1874  SDValue Offset = LD->getOffset();
1875  assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1876  Align Alignment = LD->getOriginalAlign();
1877  SDValue Mask = LD->getMask();
1878  SDValue EVL = LD->getVectorLength();
1879  EVT MemoryVT = LD->getMemoryVT();
1880 
1881  EVT LoMemVT, HiMemVT;
1882  bool HiIsEmpty = false;
1883  std::tie(LoMemVT, HiMemVT) =
1884  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1885 
1886  // Split Mask operand
1887  SDValue MaskLo, MaskHi;
1888  if (Mask.getOpcode() == ISD::SETCC) {
1889  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1890  } else {
1891  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1892  GetSplitVector(Mask, MaskLo, MaskHi);
1893  else
1894  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1895  }
1896 
1897  // Split EVL operand
1898  SDValue EVLLo, EVLHi;
1899  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
1900 
1902  LD->getPointerInfo(), MachineMemOperand::MOLoad,
1903  MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
1904 
1905  Lo =
1906  DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
1907  MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
1908 
1909  if (HiIsEmpty) {
1910  // The hi vp_load has zero storage size. We therefore simply set it to
1911  // the low vp_load and rely on subsequent removal from the chain.
1912  Hi = Lo;
1913  } else {
1914  // Generate hi vp_load.
1915  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1916  LD->isExpandingLoad());
1917 
1918  MachinePointerInfo MPI;
1919  if (LoMemVT.isScalableVector())
1920  MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1921  else
1922  MPI = LD->getPointerInfo().getWithOffset(
1923  LoMemVT.getStoreSize().getFixedSize());
1924 
1927  LD->getAAInfo(), LD->getRanges());
1928 
1929  Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
1930  Offset, MaskHi, EVLHi, HiMemVT, MMO,
1931  LD->isExpandingLoad());
1932  }
1933 
1934  // Build a factor node to remember that this load is independent of the
1935  // other one.
1936  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1937  Hi.getValue(1));
1938 
1939  // Legalize the chain result - switch anything that used the old chain to
1940  // use the new one.
1941  ReplaceValueWith(SDValue(LD, 1), Ch);
1942 }
1943 
1944 void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
1945  SDValue &Lo, SDValue &Hi) {
1946  assert(SLD->isUnindexed() &&
1947  "Indexed VP strided load during type legalization!");
1948  assert(SLD->getOffset().isUndef() &&
1949  "Unexpected indexed variable-length load offset");
1950 
1951  SDLoc DL(SLD);
1952 
1953  EVT LoVT, HiVT;
1954  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0));
1955 
1956  EVT LoMemVT, HiMemVT;
1957  bool HiIsEmpty = false;
1958  std::tie(LoMemVT, HiMemVT) =
1959  DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty);
1960 
1961  SDValue Mask = SLD->getMask();
1962  SDValue LoMask, HiMask;
1963  if (Mask.getOpcode() == ISD::SETCC) {
1964  SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
1965  } else {
1966  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1967  GetSplitVector(Mask, LoMask, HiMask);
1968  else
1969  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1970  }
1971 
1972  SDValue LoEVL, HiEVL;
1973  std::tie(LoEVL, HiEVL) =
1974  DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL);
1975 
1976  // Generate the low vp_strided_load
1977  Lo = DAG.getStridedLoadVP(
1978  SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL,
1979  SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(),
1980  LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad());
1981 
1982  if (HiIsEmpty) {
1983  // The high vp_strided_load has zero storage size. We therefore simply set
1984  // it to the low vp_strided_load and rely on subsequent removal from the
1985  // chain.
1986  Hi = Lo;
1987  } else {
1988  // Generate the high vp_strided_load.
1989  // To calculate the high base address, we need to sum to the low base
1990  // address stride number of bytes for each element already loaded by low,
1991  // that is: Ptr = Ptr + (LoEVL * Stride)
1992  EVT PtrVT = SLD->getBasePtr().getValueType();
1993  SDValue Increment =
1994  DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
1995  DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT));
1996  SDValue Ptr =
1997  DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment);
1998 
1999  Align Alignment = SLD->getOriginalAlign();
2000  if (LoMemVT.isScalableVector())
2001  Alignment = commonAlignment(
2002  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
2003 
2007  SLD->getAAInfo(), SLD->getRanges());
2008 
2010  HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
2011  SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO,
2012  SLD->isExpandingLoad());
2013  }
2014 
2015  // Build a factor node to remember that this load is independent of the
2016  // other one.
2017  SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
2018  Hi.getValue(1));
2019 
2020  // Legalize the chain result - switch anything that used the old chain to
2021  // use the new one.
2022  ReplaceValueWith(SDValue(SLD, 1), Ch);
2023 }
2024 
2025 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2026  SDValue &Lo, SDValue &Hi) {
2027  assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2028  EVT LoVT, HiVT;
2029  SDLoc dl(MLD);
2030  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
2031 
2032  SDValue Ch = MLD->getChain();
2033  SDValue Ptr = MLD->getBasePtr();
2034  SDValue Offset = MLD->getOffset();
2035  assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2036  SDValue Mask = MLD->getMask();
2037  SDValue PassThru = MLD->getPassThru();
2038  Align Alignment = MLD->getOriginalAlign();
2039  ISD::LoadExtType ExtType = MLD->getExtensionType();
2040 
2041  // Split Mask operand
2042  SDValue MaskLo, MaskHi;
2043  if (Mask.getOpcode() == ISD::SETCC) {
2044  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2045  } else {
2046  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2047  GetSplitVector(Mask, MaskLo, MaskHi);
2048  else
2049  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2050  }
2051 
2052  EVT MemoryVT = MLD->getMemoryVT();
2053  EVT LoMemVT, HiMemVT;
2054  bool HiIsEmpty = false;
2055  std::tie(LoMemVT, HiMemVT) =
2056  DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2057 
2058  SDValue PassThruLo, PassThruHi;
2059  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2060  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2061  else
2062  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2063 
2066  MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
2067  MLD->getRanges());
2068 
2069  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
2070  MMO, MLD->getAddressingMode(), ExtType,
2071  MLD->isExpandingLoad());
2072 
2073  if (HiIsEmpty) {
2074  // The hi masked load has zero storage size. We therefore simply set it to
2075  // the low masked load and rely on subsequent removal from the chain.
2076  Hi = Lo;
2077  } else {
2078  // Generate hi masked load.
2079  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2080  MLD->isExpandingLoad());
2081 
2082  MachinePointerInfo MPI;
2083  if (LoMemVT.isScalableVector())
2085  else
2086  MPI = MLD->getPointerInfo().getWithOffset(
2087  LoMemVT.getStoreSize().getFixedSize());
2088 
2091  MLD->getAAInfo(), MLD->getRanges());
2092 
2093  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
2094  HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
2095  MLD->isExpandingLoad());
2096  }
2097 
2098  // Build a factor node to remember that this load is independent of the
2099  // other one.
2100  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2101  Hi.getValue(1));
2102 
2103  // Legalize the chain result - switch anything that used the old chain to
2104  // use the new one.
2105  ReplaceValueWith(SDValue(MLD, 1), Ch);
2106 
2107 }
2108 
2109 void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2110  SDValue &Hi, bool SplitSETCC) {
2111  EVT LoVT, HiVT;
2112  SDLoc dl(N);
2113  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2114 
2115  SDValue Ch = N->getChain();
2116  SDValue Ptr = N->getBasePtr();
2117  struct Operands {
2118  SDValue Mask;
2119  SDValue Index;
2120  SDValue Scale;
2121  } Ops = [&]() -> Operands {
2122  if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2123  return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2124  }
2125  auto *VPSC = cast<VPGatherSDNode>(N);
2126  return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2127  }();
2128 
2129  EVT MemoryVT = N->getMemoryVT();
2130  Align Alignment = N->getOriginalAlign();
2131 
2132  // Split Mask operand
2133  SDValue MaskLo, MaskHi;
2134  if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2135  SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2136  } else {
2137  std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2138  }
2139 
2140  EVT LoMemVT, HiMemVT;
2141  // Split MemoryVT
2142  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2143 
2144  SDValue IndexHi, IndexLo;
2145  if (getTypeAction(Ops.Index.getValueType()) ==
2147  GetSplitVector(Ops.Index, IndexLo, IndexHi);
2148  else
2149  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2150 
2152  N->getPointerInfo(), MachineMemOperand::MOLoad,
2153  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2154 
2155  if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2156  SDValue PassThru = MGT->getPassThru();
2157  SDValue PassThruLo, PassThruHi;
2158  if (getTypeAction(PassThru.getValueType()) ==
2160  GetSplitVector(PassThru, PassThruLo, PassThruHi);
2161  else
2162  std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2163 
2164  ISD::LoadExtType ExtType = MGT->getExtensionType();
2165  ISD::MemIndexType IndexTy = MGT->getIndexType();
2166 
2167  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2168  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2169  OpsLo, MMO, IndexTy, ExtType);
2170 
2171  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2172  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2173  OpsHi, MMO, IndexTy, ExtType);
2174  } else {
2175  auto *VPGT = cast<VPGatherSDNode>(N);
2176  SDValue EVLLo, EVLHi;
2177  std::tie(EVLLo, EVLHi) =
2178  DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2179 
2180  SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2181  Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2182  MMO, VPGT->getIndexType());
2183 
2184  SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2185  Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2186  MMO, VPGT->getIndexType());
2187  }
2188 
2189  // Build a factor node to remember that this load is independent of the
2190  // other one.
2191  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2192  Hi.getValue(1));
2193 
2194  // Legalize the chain result - switch anything that used the old chain to
2195  // use the new one.
2196  ReplaceValueWith(SDValue(N, 1), Ch);
2197 }
2198 
2199 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2200  assert(N->getValueType(0).isVector() &&
2201  N->getOperand(0).getValueType().isVector() &&
2202  "Operand types must be vectors");
2203 
2204  EVT LoVT, HiVT;
2205  SDLoc DL(N);
2206  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2207 
2208  // If the input also splits, handle it directly. Otherwise split it by hand.
2209  SDValue LL, LH, RL, RH;
2210  if (getTypeAction(N->getOperand(0).getValueType()) ==
2212  GetSplitVector(N->getOperand(0), LL, LH);
2213  else
2214  std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2215 
2216  if (getTypeAction(N->getOperand(1).getValueType()) ==
2218  GetSplitVector(N->getOperand(1), RL, RH);
2219  else
2220  std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2221 
2222  if (N->getOpcode() == ISD::SETCC) {
2223  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2224  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2225  } else {
2226  assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2227  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2228  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
2229  std::tie(EVLLo, EVLHi) =
2230  DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
2231  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo,
2232  EVLLo);
2233  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi,
2234  EVLHi);
2235  }
2236 }
2237 
2238 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2239  SDValue &Hi) {
2240  // Get the dest types - they may not match the input types, e.g. int_to_fp.
2241  EVT LoVT, HiVT;
2242  SDLoc dl(N);
2243  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2244 
2245  // If the input also splits, handle it directly for a compile time speedup.
2246  // Otherwise split it by hand.
2247  EVT InVT = N->getOperand(0).getValueType();
2248  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2249  GetSplitVector(N->getOperand(0), Lo, Hi);
2250  else
2251  std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2252 
2253  const SDNodeFlags Flags = N->getFlags();
2254  unsigned Opcode = N->getOpcode();
2255  if (N->getNumOperands() <= 2) {
2256  if (Opcode == ISD::FP_ROUND) {
2257  Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2258  Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2259  } else {
2260  Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2261  Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2262  }
2263  return;
2264  }
2265 
2266  assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2267  assert(N->isVPOpcode() && "Expected VP opcode");
2268 
2269  SDValue MaskLo, MaskHi;
2270  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2271 
2272  SDValue EVLLo, EVLHi;
2273  std::tie(EVLLo, EVLHi) =
2274  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2275 
2276  Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2277  Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2278 }
2279 
2280 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2281  SDValue &Hi) {
2282  SDLoc dl(N);
2283  EVT SrcVT = N->getOperand(0).getValueType();
2284  EVT DestVT = N->getValueType(0);
2285  EVT LoVT, HiVT;
2286  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2287 
2288  // We can do better than a generic split operation if the extend is doing
2289  // more than just doubling the width of the elements and the following are
2290  // true:
2291  // - The number of vector elements is even,
2292  // - the source type is legal,
2293  // - the type of a split source is illegal,
2294  // - the type of an extended (by doubling element size) source is legal, and
2295  // - the type of that extended source when split is legal.
2296  //
2297  // This won't necessarily completely legalize the operation, but it will
2298  // more effectively move in the right direction and prevent falling down
2299  // to scalarization in many cases due to the input vector being split too
2300  // far.
2301  if (SrcVT.getVectorElementCount().isKnownEven() &&
2302  SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2303  LLVMContext &Ctx = *DAG.getContext();
2304  EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2305  EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2306 
2307  EVT SplitLoVT, SplitHiVT;
2308  std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2309  if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2310  TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2311  LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2312  N->dump(&DAG); dbgs() << "\n");
2313  if (!N->isVPOpcode()) {
2314  // Extend the source vector by one step.
2315  SDValue NewSrc =
2316  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2317  // Get the low and high halves of the new, extended one step, vector.
2318  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2319  // Extend those vector halves the rest of the way.
2320  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2321  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2322  return;
2323  }
2324 
2325  // Extend the source vector by one step.
2326  SDValue NewSrc =
2327  DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0),
2328  N->getOperand(1), N->getOperand(2));
2329  // Get the low and high halves of the new, extended one step, vector.
2330  std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2331 
2332  SDValue MaskLo, MaskHi;
2333  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2334 
2335  SDValue EVLLo, EVLHi;
2336  std::tie(EVLLo, EVLHi) =
2337  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2338  // Extend those vector halves the rest of the way.
2339  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2340  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2341  return;
2342  }
2343  }
2344  // Fall back to the generic unary operator splitting otherwise.
2345  SplitVecRes_UnaryOp(N, Lo, Hi);
2346 }
2347 
2348 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2349  SDValue &Lo, SDValue &Hi) {
2350  // The low and high parts of the original input give four input vectors.
2351  SDValue Inputs[4];
2352  SDLoc DL(N);
2353  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2354  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2355  EVT NewVT = Inputs[0].getValueType();
2356  unsigned NewElts = NewVT.getVectorNumElements();
2357 
2358  auto &&IsConstant = [](const SDValue &N) {
2359  APInt SplatValue;
2360  return N.getResNo() == 0 &&
2361  (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2363  };
2364  auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2365  SDValue &Input2,
2366  ArrayRef<int> Mask) {
2367  assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2368  Input2->getOpcode() == ISD::BUILD_VECTOR &&
2369  "Expected build vector node.");
2370  EVT EltVT = NewVT.getVectorElementType();
2371  SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2372  for (unsigned I = 0; I < NewElts; ++I) {
2373  if (Mask[I] == UndefMaskElem)
2374  continue;
2375  unsigned Idx = Mask[I];
2376  if (Idx >= NewElts)
2377  Ops[I] = Input2.getOperand(Idx - NewElts);
2378  else
2379  Ops[I] = Input1.getOperand(Idx);
2380  // Make the type of all elements the same as the element type.
2381  if (Ops[I].getValueType().bitsGT(EltVT))
2382  Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2383  }
2384  return DAG.getBuildVector(NewVT, DL, Ops);
2385  };
2386 
2387  // If Lo or Hi uses elements from at most two of the four input vectors, then
2388  // express it as a vector shuffle of those two inputs. Otherwise extract the
2389  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2390  SmallVector<int> OrigMask(N->getMask());
2391  // Try to pack incoming shuffles/inputs.
2392  auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2394  // Check if all inputs are shuffles of the same operands or non-shuffles.
2396  for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2397  SDValue Input = Inputs[Idx];
2398  auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2399  if (!Shuffle ||
2400  Input.getOperand(0).getValueType() != Input.getValueType())
2401  continue;
2402  ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2403  .push_back(Idx);
2404  ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2405  .push_back(Idx);
2406  }
2407  for (auto &P : ShufflesIdxs) {
2408  if (P.second.size() < 2)
2409  continue;
2410  // Use shuffles operands instead of shuffles themselves.
2411  // 1. Adjust mask.
2412  for (int &Idx : Mask) {
2413  if (Idx == UndefMaskElem)
2414  continue;
2415  unsigned SrcRegIdx = Idx / NewElts;
2416  if (Inputs[SrcRegIdx].isUndef()) {
2417  Idx = UndefMaskElem;
2418  continue;
2419  }
2420  auto *Shuffle =
2421  dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2422  if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2423  continue;
2424  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2425  if (MaskElt == UndefMaskElem) {
2426  Idx = UndefMaskElem;
2427  continue;
2428  }
2429  Idx = MaskElt % NewElts +
2430  P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2431  ? 0
2432  : 1] *
2433  NewElts;
2434  }
2435  // 2. Update inputs.
2436  Inputs[P.second[0]] = P.first.first;
2437  Inputs[P.second[1]] = P.first.second;
2438  // Clear the pair data.
2439  P.second.clear();
2440  ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2441  }
2442  // Check if any concat_vectors can be simplified.
2443  SmallBitVector UsedSubVector(2 * std::size(Inputs));
2444  for (int &Idx : Mask) {
2445  if (Idx == UndefMaskElem)
2446  continue;
2447  unsigned SrcRegIdx = Idx / NewElts;
2448  if (Inputs[SrcRegIdx].isUndef()) {
2449  Idx = UndefMaskElem;
2450  continue;
2451  }
2453  getTypeAction(Inputs[SrcRegIdx].getValueType());
2454  if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2455  Inputs[SrcRegIdx].getNumOperands() == 2 &&
2456  !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2457  (TypeAction == TargetLowering::TypeLegal ||
2458  TypeAction == TargetLowering::TypeWidenVector))
2459  UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2460  }
2461  if (UsedSubVector.count() > 1) {
2463  for (unsigned I = 0; I < std::size(Inputs); ++I) {
2464  if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2465  continue;
2466  if (Pairs.empty() || Pairs.back().size() == 2)
2467  Pairs.emplace_back();
2468  if (UsedSubVector.test(2 * I)) {
2469  Pairs.back().emplace_back(I, 0);
2470  } else {
2471  assert(UsedSubVector.test(2 * I + 1) &&
2472  "Expected to be used one of the subvectors.");
2473  Pairs.back().emplace_back(I, 1);
2474  }
2475  }
2476  if (!Pairs.empty() && Pairs.front().size() > 1) {
2477  // Adjust mask.
2478  for (int &Idx : Mask) {
2479  if (Idx == UndefMaskElem)
2480  continue;
2481  unsigned SrcRegIdx = Idx / NewElts;
2482  auto *It = find_if(
2483  Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2484  return Idxs.front().first == SrcRegIdx ||
2485  Idxs.back().first == SrcRegIdx;
2486  });
2487  if (It == Pairs.end())
2488  continue;
2489  Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2490  (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2491  }
2492  // Adjust inputs.
2493  for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2494  Inputs[Idxs.front().first] = DAG.getNode(
2496  Inputs[Idxs.front().first].getValueType(),
2497  Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2498  Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2499  }
2500  }
2501  }
2502  bool Changed;
2503  do {
2504  // Try to remove extra shuffles (except broadcasts) and shuffles with the
2505  // reused operands.
2506  Changed = false;
2507  for (unsigned I = 0; I < std::size(Inputs); ++I) {
2508  auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2509  if (!Shuffle)
2510  continue;
2511  if (Shuffle->getOperand(0).getValueType() != NewVT)
2512  continue;
2513  int Op = -1;
2514  if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2515  !Shuffle->isSplat()) {
2516  Op = 0;
2517  } else if (!Inputs[I].hasOneUse() &&
2518  !Shuffle->getOperand(1).isUndef()) {
2519  // Find the only used operand, if possible.
2520  for (int &Idx : Mask) {
2521  if (Idx == UndefMaskElem)
2522  continue;
2523  unsigned SrcRegIdx = Idx / NewElts;
2524  if (SrcRegIdx != I)
2525  continue;
2526  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2527  if (MaskElt == UndefMaskElem) {
2528  Idx = UndefMaskElem;
2529  continue;
2530  }
2531  int OpIdx = MaskElt / NewElts;
2532  if (Op == -1) {
2533  Op = OpIdx;
2534  continue;
2535  }
2536  if (Op != OpIdx) {
2537  Op = -1;
2538  break;
2539  }
2540  }
2541  }
2542  if (Op < 0) {
2543  // Try to check if one of the shuffle operands is used already.
2544  for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2545  if (Shuffle->getOperand(OpIdx).isUndef())
2546  continue;
2547  auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2548  if (It == std::end(Inputs))
2549  continue;
2550  int FoundOp = std::distance(std::begin(Inputs), It);
2551  // Found that operand is used already.
2552  // 1. Fix the mask for the reused operand.
2553  for (int &Idx : Mask) {
2554  if (Idx == UndefMaskElem)
2555  continue;
2556  unsigned SrcRegIdx = Idx / NewElts;
2557  if (SrcRegIdx != I)
2558  continue;
2559  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2560  if (MaskElt == UndefMaskElem) {
2561  Idx = UndefMaskElem;
2562  continue;
2563  }
2564  int MaskIdx = MaskElt / NewElts;
2565  if (OpIdx == MaskIdx)
2566  Idx = MaskElt % NewElts + FoundOp * NewElts;
2567  }
2568  // 2. Set Op to the unused OpIdx.
2569  Op = (OpIdx + 1) % 2;
2570  break;
2571  }
2572  }
2573  if (Op >= 0) {
2574  Changed = true;
2575  Inputs[I] = Shuffle->getOperand(Op);
2576  // Adjust mask.
2577  for (int &Idx : Mask) {
2578  if (Idx == UndefMaskElem)
2579  continue;
2580  unsigned SrcRegIdx = Idx / NewElts;
2581  if (SrcRegIdx != I)
2582  continue;
2583  int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2584  int OpIdx = MaskElt / NewElts;
2585  if (OpIdx != Op)
2586  continue;
2587  Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2588  }
2589  }
2590  }
2591  } while (Changed);
2592  };
2593  TryPeekThroughShufflesInputs(OrigMask);
2594  // Proces unique inputs.
2595  auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2596  NewElts](SmallVectorImpl<int> &Mask) {
2597  SetVector<SDValue> UniqueInputs;
2598  SetVector<SDValue> UniqueConstantInputs;
2599  for (const auto &I : Inputs) {
2600  if (IsConstant(I))
2601  UniqueConstantInputs.insert(I);
2602  else if (!I.isUndef())
2603  UniqueInputs.insert(I);
2604  }
2605  // Adjust mask in case of reused inputs. Also, need to insert constant
2606  // inputs at first, otherwise it affects the final outcome.
2607  if (UniqueInputs.size() != std::size(Inputs)) {
2608  auto &&UniqueVec = UniqueInputs.takeVector();
2609  auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2610  unsigned ConstNum = UniqueConstantVec.size();
2611  for (int &Idx : Mask) {
2612  if (Idx == UndefMaskElem)
2613  continue;
2614  unsigned SrcRegIdx = Idx / NewElts;
2615  if (Inputs[SrcRegIdx].isUndef()) {
2616  Idx = UndefMaskElem;
2617  continue;
2618  }
2619  const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2620  if (It != UniqueConstantVec.end()) {
2621  Idx = (Idx % NewElts) +
2622  NewElts * std::distance(UniqueConstantVec.begin(), It);
2623  assert(Idx >= 0 && "Expected defined mask idx.");
2624  continue;
2625  }
2626  const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2627  assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2628  Idx = (Idx % NewElts) +
2629  NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2630  assert(Idx >= 0 && "Expected defined mask idx.");
2631  }
2632  copy(UniqueConstantVec, std::begin(Inputs));
2633  copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2634  }
2635  };
2636  MakeUniqueInputs(OrigMask);
2637  SDValue OrigInputs[4];
2638  copy(Inputs, std::begin(OrigInputs));
2639  for (unsigned High = 0; High < 2; ++High) {
2640  SDValue &Output = High ? Hi : Lo;
2641 
2642  // Build a shuffle mask for the output, discovering on the fly which
2643  // input vectors to use as shuffle operands.
2644  unsigned FirstMaskIdx = High * NewElts;
2645  SmallVector<int> Mask(NewElts * std::size(Inputs), UndefMaskElem);
2646  copy(makeArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2647  assert(!Output && "Expected default initialized initial value.");
2648  TryPeekThroughShufflesInputs(Mask);
2649  MakeUniqueInputs(Mask);
2650  SDValue TmpInputs[4];
2651  copy(Inputs, std::begin(TmpInputs));
2652  // Track changes in the output registers.
2653  int UsedIdx = -1;
2654  bool SecondIteration = false;
2655  auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2656  if (UsedIdx < 0) {
2657  UsedIdx = Idx;
2658  return false;
2659  }
2660  if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2661  SecondIteration = true;
2662  return SecondIteration;
2663  };
2665  Mask, std::size(Inputs), std::size(Inputs),
2666  /*NumOfUsedRegs=*/1,
2667  [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2668  [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2669  &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2670  if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2671  Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2672  else
2673  Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2674  DAG.getUNDEF(NewVT), Mask);
2675  Inputs[Idx] = Output;
2676  },
2677  [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2678  &TmpInputs,
2679  &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2680  if (AccumulateResults(Idx1)) {
2681  if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2682  Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2683  Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2684  else
2685  Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2686  Inputs[Idx2], Mask);
2687  } else {
2688  if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2689  TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2690  Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2691  else
2692  Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2693  TmpInputs[Idx2], Mask);
2694  }
2695  Inputs[Idx1] = Output;
2696  });
2697  copy(OrigInputs, std::begin(Inputs));
2698  }
2699 }
2700 
2701 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2702  EVT OVT = N->getValueType(0);
2703  EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2704  SDValue Chain = N->getOperand(0);
2705  SDValue Ptr = N->getOperand(1);
2706  SDValue SV = N->getOperand(2);
2707  SDLoc dl(N);
2708 
2709  const Align Alignment =
2710  DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2711 
2712  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2713  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2714  Chain = Hi.getValue(1);
2715 
2716  // Modified the chain - switch anything that used the old chain to use
2717  // the new one.
2718  ReplaceValueWith(SDValue(N, 1), Chain);
2719 }
2720 
2721 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2722  SDValue &Hi) {
2723  EVT DstVTLo, DstVTHi;
2724  std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2725  SDLoc dl(N);
2726 
2727  SDValue SrcLo, SrcHi;
2728  EVT SrcVT = N->getOperand(0).getValueType();
2729  if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2730  GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2731  else
2732  std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2733 
2734  Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2735  Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2736 }
2737 
2738 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2739  SDValue &Hi) {
2740  SDValue InLo, InHi;
2741  GetSplitVector(N->getOperand(0), InLo, InHi);
2742  SDLoc DL(N);
2743 
2744  Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2745  Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2746 }
2747 
2748 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2749  SDValue &Hi) {
2750  EVT VT = N->getValueType(0);
2751  SDLoc DL(N);
2752 
2753  EVT LoVT, HiVT;
2754  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2755 
2756  SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2757  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
2758  DAG.getVectorIdxConstant(0, DL));
2759  Hi =
2760  DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
2761  DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2762 }
2763 
2764 //===----------------------------------------------------------------------===//
2765 // Operand Vector Splitting
2766 //===----------------------------------------------------------------------===//
2767 
2768 /// This method is called when the specified operand of the specified node is
2769 /// found to need vector splitting. At this point, all of the result types of
2770 /// the node are known to be legal, but other operands of the node may need
2771 /// legalization as well as the specified one.
2772 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2773  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2774  SDValue Res = SDValue();
2775 
2776  // See if the target wants to custom split this node.
2777  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2778  return false;
2779 
2780  switch (N->getOpcode()) {
2781  default:
2782 #ifndef NDEBUG
2783  dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2784  N->dump(&DAG);
2785  dbgs() << "\n";
2786 #endif
2787  report_fatal_error("Do not know how to split this operator's "
2788  "operand!\n");
2789 
2790  case ISD::VP_SETCC:
2791  case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2792  case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2793  case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2794  case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2795  case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2796  case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2797  case ISD::VP_TRUNCATE:
2798  case ISD::TRUNCATE:
2799  Res = SplitVecOp_TruncateHelper(N);
2800  break;
2801  case ISD::STRICT_FP_ROUND:
2802  case ISD::VP_FP_ROUND:
2803  case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2804  case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2805  case ISD::STORE:
2806  Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2807  break;
2808  case ISD::VP_STORE:
2809  Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2810  break;
2811  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2812  Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo);
2813  break;
2814  case ISD::MSTORE:
2815  Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2816  break;
2817  case ISD::MSCATTER:
2818  case ISD::VP_SCATTER:
2819  Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
2820  break;
2821  case ISD::MGATHER:
2822  case ISD::VP_GATHER:
2823  Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
2824  break;
2825  case ISD::VSELECT:
2826  Res = SplitVecOp_VSELECT(N, OpNo);
2827  break;
2830  case ISD::SINT_TO_FP:
2831  case ISD::UINT_TO_FP:
2832  case ISD::VP_SINT_TO_FP:
2833  case ISD::VP_UINT_TO_FP:
2834  if (N->getValueType(0).bitsLT(
2835  N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2836  Res = SplitVecOp_TruncateHelper(N);
2837  else
2838  Res = SplitVecOp_UnaryOp(N);
2839  break;
2840  case ISD::FP_TO_SINT_SAT:
2841  case ISD::FP_TO_UINT_SAT:
2842  Res = SplitVecOp_FP_TO_XINT_SAT(N);
2843  break;
2844  case ISD::FP_TO_SINT:
2845  case ISD::FP_TO_UINT:
2846  case ISD::VP_FP_TO_SINT:
2847  case ISD::VP_FP_TO_UINT:
2850  case ISD::STRICT_FP_EXTEND:
2851  case ISD::FP_EXTEND:
2852  case ISD::SIGN_EXTEND:
2853  case ISD::ZERO_EXTEND:
2854  case ISD::ANY_EXTEND:
2855  case ISD::FTRUNC:
2856  Res = SplitVecOp_UnaryOp(N);
2857  break;
2858 
2862  Res = SplitVecOp_ExtVecInRegOp(N);
2863  break;
2864 
2865  case ISD::VECREDUCE_FADD:
2866  case ISD::VECREDUCE_FMUL:
2867  case ISD::VECREDUCE_ADD:
2868  case ISD::VECREDUCE_MUL:
2869  case ISD::VECREDUCE_AND:
2870  case ISD::VECREDUCE_OR:
2871  case ISD::VECREDUCE_XOR:
2872  case ISD::VECREDUCE_SMAX:
2873  case ISD::VECREDUCE_SMIN:
2874  case ISD::VECREDUCE_UMAX:
2875  case ISD::VECREDUCE_UMIN:
2876  case ISD::VECREDUCE_FMAX:
2877  case ISD::VECREDUCE_FMIN:
2878  Res = SplitVecOp_VECREDUCE(N, OpNo);
2879  break;
2882  Res = SplitVecOp_VECREDUCE_SEQ(N);
2883  break;
2884  case ISD::VP_REDUCE_FADD:
2885  case ISD::VP_REDUCE_SEQ_FADD:
2886  case ISD::VP_REDUCE_FMUL:
2887  case ISD::VP_REDUCE_SEQ_FMUL:
2888  case ISD::VP_REDUCE_ADD:
2889  case ISD::VP_REDUCE_MUL:
2890  case ISD::VP_REDUCE_AND:
2891  case ISD::VP_REDUCE_OR:
2892  case ISD::VP_REDUCE_XOR:
2893  case ISD::VP_REDUCE_SMAX:
2894  case ISD::VP_REDUCE_SMIN:
2895  case ISD::VP_REDUCE_UMAX:
2896  case ISD::VP_REDUCE_UMIN:
2897  case ISD::VP_REDUCE_FMAX:
2898  case ISD::VP_REDUCE_FMIN:
2899  Res = SplitVecOp_VP_REDUCE(N, OpNo);
2900  break;
2901  }
2902 
2903  // If the result is null, the sub-method took care of registering results etc.
2904  if (!Res.getNode()) return false;
2905 
2906  // If the result is N, the sub-method updated N in place. Tell the legalizer
2907  // core about this.
2908  if (Res.getNode() == N)
2909  return true;
2910 
2911  if (N->isStrictFPOpcode())
2912  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2913  "Invalid operand expansion");
2914  else
2915  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2916  "Invalid operand expansion");
2917 
2918  ReplaceValueWith(SDValue(N, 0), Res);
2919  return false;
2920 }
2921 
2922 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2923  // The only possibility for an illegal operand is the mask, since result type
2924  // legalization would have handled this node already otherwise.
2925  assert(OpNo == 0 && "Illegal operand must be mask");
2926 
2927  SDValue Mask = N->getOperand(0);
2928  SDValue Src0 = N->getOperand(1);
2929  SDValue Src1 = N->getOperand(2);
2930  EVT Src0VT = Src0.getValueType();
2931  SDLoc DL(N);
2932  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2933 
2934  SDValue Lo, Hi;
2935  GetSplitVector(N->getOperand(0), Lo, Hi);
2936  assert(Lo.getValueType() == Hi.getValueType() &&
2937  "Lo and Hi have differing types");
2938 
2939  EVT LoOpVT, HiOpVT;
2940  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2941  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2942 
2943  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2944  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2945  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2946  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2947 
2948  SDValue LoSelect =
2949  DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2950  SDValue HiSelect =
2951  DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2952 
2953  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2954 }
2955 
2956 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2957  EVT ResVT = N->getValueType(0);
2958  SDValue Lo, Hi;
2959  SDLoc dl(N);
2960 
2961  SDValue VecOp = N->getOperand(OpNo);
2962  EVT VecVT = VecOp.getValueType();
2963  assert(VecVT.isVector() && "Can only split reduce vector operand");
2964  GetSplitVector(VecOp, Lo, Hi);
2965  EVT LoOpVT, HiOpVT;
2966  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2967 
2968  // Use the appropriate scalar instruction on the split subvectors before
2969  // reducing the now partially reduced smaller vector.
2970  unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2971  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2972  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2973 }
2974 
2975 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2976  EVT ResVT = N->getValueType(0);
2977  SDValue Lo, Hi;
2978  SDLoc dl(N);
2979 
2980  SDValue AccOp = N->getOperand(0);
2981  SDValue VecOp = N->getOperand(1);
2982  SDNodeFlags Flags = N->getFlags();
2983 
2984  EVT VecVT = VecOp.getValueType();
2985  assert(VecVT.isVector() && "Can only split reduce vector operand");
2986  GetSplitVector(VecOp, Lo, Hi);
2987  EVT LoOpVT, HiOpVT;
2988  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2989 
2990  // Reduce low half.
2991  SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2992 
2993  // Reduce high half, using low half result as initial value.
2994  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2995 }
2996 
2997 SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2998  assert(N->isVPOpcode() && "Expected VP opcode");
2999  assert(OpNo == 1 && "Can only split reduce vector operand");
3000 
3001  unsigned Opc = N->getOpcode();
3002  EVT ResVT = N->getValueType(0);
3003  SDValue Lo, Hi;
3004  SDLoc dl(N);
3005 
3006  SDValue VecOp = N->getOperand(OpNo);
3007  EVT VecVT = VecOp.getValueType();
3008  assert(VecVT.isVector() && "Can only split reduce vector operand");
3009  GetSplitVector(VecOp, Lo, Hi);
3010 
3011  SDValue MaskLo, MaskHi;
3012  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
3013 
3014  SDValue EVLLo, EVLHi;
3015  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
3016 
3017  const SDNodeFlags Flags = N->getFlags();
3018 
3019  SDValue ResLo =
3020  DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
3021  return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
3022 }
3023 
3024 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3025  // The result has a legal vector type, but the input needs splitting.
3026  EVT ResVT = N->getValueType(0);
3027  SDValue Lo, Hi;
3028  SDLoc dl(N);
3029  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3030  EVT InVT = Lo.getValueType();
3031 
3032  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3033  InVT.getVectorElementCount());
3034 
3035  if (N->isStrictFPOpcode()) {
3036  Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3037  { N->getOperand(0), Lo });
3038  Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3039  { N->getOperand(0), Hi });
3040 
3041  // Build a factor node to remember that this operation is independent
3042  // of the other one.
3043  SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3044  Hi.getValue(1));
3045 
3046  // Legalize the chain result - switch anything that used the old chain to
3047  // use the new one.
3048  ReplaceValueWith(SDValue(N, 1), Ch);
3049  } else if (N->getNumOperands() == 3) {
3050  assert(N->isVPOpcode() && "Expected VP opcode");
3051  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3052  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3053  std::tie(EVLLo, EVLHi) =
3054  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
3055  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo);
3056  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi);
3057  } else {
3058  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
3059  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
3060  }
3061 
3062  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3063 }
3064 
3065 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3066  // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3067  // end up being split all the way down to individual components. Convert the
3068  // split pieces into integers and reassemble.
3069  SDValue Lo, Hi;
3070  GetSplitVector(N->getOperand(0), Lo, Hi);
3071  Lo = BitConvertToInteger(Lo);
3072  Hi = BitConvertToInteger(Hi);
3073 
3074  if (DAG.getDataLayout().isBigEndian())
3075  std::swap(Lo, Hi);
3076 
3077  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
3078  JoinIntegers(Lo, Hi));
3079 }
3080 
3081 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3082  unsigned OpNo) {
3083  assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3084  // We know that the result type is legal.
3085  EVT ResVT = N->getValueType(0);
3086 
3087  SDValue Vec = N->getOperand(0);
3088  SDValue SubVec = N->getOperand(1);
3089  SDValue Idx = N->getOperand(2);
3090  SDLoc dl(N);
3091 
3092  SDValue Lo, Hi;
3093  GetSplitVector(SubVec, Lo, Hi);
3094 
3095  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3096  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3097 
3098  SDValue FirstInsertion =
3099  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
3100  SDValue SecondInsertion =
3101  DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
3102  DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3103 
3104  return SecondInsertion;
3105 }
3106 
3107 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3108  // We know that the extracted result type is legal.
3109  EVT SubVT = N->getValueType(0);
3110  SDValue Idx = N->getOperand(1);
3111  SDLoc dl(N);
3112  SDValue Lo, Hi;
3113 
3114  GetSplitVector(N->getOperand(0), Lo, Hi);
3115 
3116  uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3117  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3118 
3119  if (IdxVal < LoEltsMin) {
3120  assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3121  "Extracted subvector crosses vector split!");
3122  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
3123  } else if (SubVT.isScalableVector() ==
3124  N->getOperand(0).getValueType().isScalableVector())
3125  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
3126  DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3127 
3128  // After this point the DAG node only permits extracting fixed-width
3129  // subvectors from scalable vectors.
3130  assert(SubVT.isFixedLengthVector() &&
3131  "Extracting scalable subvector from fixed-width unsupported");
3132 
3133  // If the element type is i1 and we're not promoting the result, then we may
3134  // end up loading the wrong data since the bits are packed tightly into
3135  // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3136  // type at index 4, then we will load a byte starting at index 0.
3137  if (SubVT.getScalarType() == MVT::i1)
3138  report_fatal_error("Don't know how to extract fixed-width predicate "
3139  "subvector from a scalable predicate vector");
3140 
3141  // Spill the vector to the stack. We should use the alignment for
3142  // the smallest part.
3143  SDValue Vec = N->getOperand(0);
3144  EVT VecVT = Vec.getValueType();
3145  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3146  SDValue StackPtr =
3147  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3148  auto &MF = DAG.getMachineFunction();
3149  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3150  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3151 
3152  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3153  SmallestAlign);
3154 
3155  // Extract the subvector by loading the correct part.
3156  StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
3157 
3158  return DAG.getLoad(
3159  SubVT, dl, Store, StackPtr,
3160  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3161 }
3162 
3163 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3164  SDValue Vec = N->getOperand(0);
3165  SDValue Idx = N->getOperand(1);
3166  EVT VecVT = Vec.getValueType();
3167 
3168  if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) {
3169  uint64_t IdxVal = Index->getZExtValue();
3170 
3171  SDValue Lo, Hi;
3172  GetSplitVector(Vec, Lo, Hi);
3173 
3174  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3175 
3176  if (IdxVal < LoElts)
3177  return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
3178  else if (!Vec.getValueType().isScalableVector())
3179  return SDValue(DAG.UpdateNodeOperands(N, Hi,
3180  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
3181  Idx.getValueType())), 0);
3182  }
3183 
3184  // See if the target wants to custom expand this node.
3185  if (CustomLowerNode(N, N->getValueType(0), true))
3186  return SDValue();
3187 
3188  // Make the vector elements byte-addressable if they aren't already.
3189  SDLoc dl(N);
3190  EVT EltVT = VecVT.getVectorElementType();
3191  if (VecVT.getScalarSizeInBits() < 8) {
3192  EltVT = MVT::i8;
3193  VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3194  VecVT.getVectorElementCount());
3195  Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3196  }
3197 
3198  // Store the vector to the stack.
3199  // In cases where the vector is illegal it will be broken down into parts
3200  // and stored in parts - we should use the alignment for the smallest part.
3201  Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3202  SDValue StackPtr =
3203  DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3204  auto &MF = DAG.getMachineFunction();
3205  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3206  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3207  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3208  SmallestAlign);
3209 
3210  // Load back the required element.
3211  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3212 
3213  // FIXME: This is to handle i1 vectors with elements promoted to i8.
3214  // i1 vector handling needs general improvement.
3215  if (N->getValueType(0).bitsLT(EltVT)) {
3216  SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3217  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3218  return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3219  }
3220 
3221  return DAG.getExtLoad(
3222  ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3223  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3224  commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3225 }
3226 
3227 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3228  SDValue Lo, Hi;
3229 
3230  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3231  // splitting the result has the same effect as splitting the input operand.
3232  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3233 
3234  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3235 }
3236 
3237 SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3238  (void)OpNo;
3239  SDValue Lo, Hi;
3240  SplitVecRes_Gather(N, Lo, Hi);
3241 
3242  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3243  ReplaceValueWith(SDValue(N, 0), Res);
3244  return SDValue();
3245 }
3246 
3247 SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3248  assert(N->isUnindexed() && "Indexed vp_store of vector?");
3249  SDValue Ch = N->getChain();
3250  SDValue Ptr = N->getBasePtr();
3251  SDValue Offset = N->getOffset();
3252  assert(Offset.isUndef() && "Unexpected VP store offset");
3253  SDValue Mask = N->getMask();
3254  SDValue EVL = N->getVectorLength();
3255  SDValue Data = N->getValue();
3256  Align Alignment = N->getOriginalAlign();
3257  SDLoc DL(N);
3258 
3259  SDValue DataLo, DataHi;
3260  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3261  // Split Data operand
3262  GetSplitVector(Data, DataLo, DataHi);
3263  else
3264  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3265 
3266  // Split Mask operand
3267  SDValue MaskLo, MaskHi;
3268  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3269  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3270  } else {
3271  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3272  GetSplitVector(Mask, MaskLo, MaskHi);
3273  else
3274  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3275  }
3276 
3277  EVT MemoryVT = N->getMemoryVT();
3278  EVT LoMemVT, HiMemVT;
3279  bool HiIsEmpty = false;
3280  std::tie(LoMemVT, HiMemVT) =
3281  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3282 
3283  // Split EVL
3284  SDValue EVLLo, EVLHi;
3285  std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3286 
3287  SDValue Lo, Hi;
3288  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3289  N->getPointerInfo(), MachineMemOperand::MOStore,
3290  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3291 
3292  Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3293  N->getAddressingMode(), N->isTruncatingStore(),
3294  N->isCompressingStore());
3295 
3296  // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3297  if (HiIsEmpty)
3298  return Lo;
3299 
3300  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3301  N->isCompressingStore());
3302 
3303  MachinePointerInfo MPI;
3304  if (LoMemVT.isScalableVector()) {
3305  Alignment = commonAlignment(Alignment,
3306  LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3307  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3308  } else
3309  MPI = N->getPointerInfo().getWithOffset(
3310  LoMemVT.getStoreSize().getFixedSize());
3311 
3312  MMO = DAG.getMachineFunction().getMachineMemOperand(
3314  N->getAAInfo(), N->getRanges());
3315 
3316  Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3317  N->getAddressingMode(), N->isTruncatingStore(),
3318  N->isCompressingStore());
3319 
3320  // Build a factor node to remember that this store is independent of the
3321  // other one.
3322  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3323 }
3324 
3325 SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3326  unsigned OpNo) {
3327  assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3328  assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3329 
3330  SDLoc DL(N);
3331 
3332  SDValue Data = N->getValue();
3333  SDValue LoData, HiData;
3334  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3335  GetSplitVector(Data, LoData, HiData);
3336  else
3337  std::tie(LoData, HiData) = DAG.SplitVector(Data, DL);
3338 
3339  EVT LoMemVT, HiMemVT;
3340  bool HiIsEmpty = false;
3341  std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3342  N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty);
3343 
3344  SDValue Mask = N->getMask();
3345  SDValue LoMask, HiMask;
3346  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3347  SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
3348  else if (getTypeAction(Mask.getValueType()) ==
3350  GetSplitVector(Mask, LoMask, HiMask);
3351  else
3352  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3353 
3354  SDValue LoEVL, HiEVL;
3355  std::tie(LoEVL, HiEVL) =
3356  DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL);
3357 
3358  // Generate the low vp_strided_store
3359  SDValue Lo = DAG.getStridedStoreVP(
3360  N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(),
3361  N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(),
3362  N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore());
3363 
3364  // If the high vp_strided_store has zero storage size, only the low
3365  // vp_strided_store is needed.
3366  if (HiIsEmpty)
3367  return Lo;
3368 
3369  // Generate the high vp_strided_store.
3370  // To calculate the high base address, we need to sum to the low base
3371  // address stride number of bytes for each element already stored by low,
3372  // that is: Ptr = Ptr + (LoEVL * Stride)
3373  EVT PtrVT = N->getBasePtr().getValueType();
3374  SDValue Increment =
3375  DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
3376  DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT));
3377  SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment);
3378 
3379  Align Alignment = N->getOriginalAlign();
3380  if (LoMemVT.isScalableVector())
3381  Alignment = commonAlignment(Alignment,
3382  LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3383 
3384  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3385  MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3387  N->getAAInfo(), N->getRanges());
3388 
3389  SDValue Hi = DAG.getStridedStoreVP(
3390  N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
3391  HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(),
3392  N->isCompressingStore());
3393 
3394  // Build a factor node to remember that this store is independent of the
3395  // other one.
3396  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3397 }
3398 
3399 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3400  unsigned OpNo) {
3401  assert(N->isUnindexed() && "Indexed masked store of vector?");
3402  SDValue Ch = N->getChain();
3403  SDValue Ptr = N->getBasePtr();
3404  SDValue Offset = N->getOffset();
3405  assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3406  SDValue Mask = N->getMask();
3407  SDValue Data = N->getValue();
3408  Align Alignment = N->getOriginalAlign();
3409  SDLoc DL(N);
3410 
3411  SDValue DataLo, DataHi;
3412  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3413  // Split Data operand
3414  GetSplitVector(Data, DataLo, DataHi);
3415  else
3416  std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3417 
3418  // Split Mask operand
3419  SDValue MaskLo, MaskHi;
3420  if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3421  SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3422  } else {
3423  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3424  GetSplitVector(Mask, MaskLo, MaskHi);
3425  else
3426  std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3427  }
3428 
3429  EVT MemoryVT = N->getMemoryVT();
3430  EVT LoMemVT, HiMemVT;
3431  bool HiIsEmpty = false;
3432  std::tie(LoMemVT, HiMemVT) =
3433  DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3434 
3435  SDValue Lo, Hi, Res;
3436  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3437  N->getPointerInfo(), MachineMemOperand::MOStore,
3438  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3439 
3440  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3441  N->getAddressingMode(), N->isTruncatingStore(),
3442  N->isCompressingStore());
3443 
3444  if (HiIsEmpty) {
3445  // The hi masked store has zero storage size.
3446  // Only the lo masked store is needed.
3447  Res = Lo;
3448  } else {
3449 
3450  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3451  N->isCompressingStore());
3452 
3453  MachinePointerInfo MPI;
3454  if (LoMemVT.isScalableVector()) {
3455  Alignment = commonAlignment(
3456  Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
3457  MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3458  } else
3459  MPI = N->getPointerInfo().getWithOffset(
3460  LoMemVT.getStoreSize().getFixedSize());
3461 
3462  MMO = DAG.getMachineFunction().getMachineMemOperand(
3464  N->getAAInfo(), N->getRanges());
3465 
3466  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3467  N->getAddressingMode(), N->isTruncatingStore(),
3468  N->isCompressingStore());
3469 
3470  // Build a factor node to remember that this store is independent of the
3471  // other one.
3472  Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3473  }
3474 
3475  return Res;
3476 }
3477 
3478 SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3479  SDValue Ch = N->getChain();
3480  SDValue Ptr = N->getBasePtr();
3481  EVT MemoryVT = N->getMemoryVT();
3482  Align Alignment = N->getOriginalAlign();
3483  SDLoc DL(N);
3484  struct Operands {
3485  SDValue Mask;
3486  SDValue Index;
3487  SDValue Scale;
3488  SDValue Data;
3489  } Ops = [&]() -> Operands {
3490  if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3491  return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3492  MSC->getValue()};
3493  }
3494  auto *VPSC = cast<VPScatterSDNode>(N);
3495  return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3496  VPSC->getValue()};
3497  }();
3498  // Split all operands
3499 
3500  EVT LoMemVT, HiMemVT;
3501  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3502 
3503  SDValue DataLo, DataHi;
3504  if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3505  // Split Data operand
3506  GetSplitVector(Ops.Data, DataLo, DataHi);
3507  else
3508  std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3509 
3510  // Split Mask operand
3511  SDValue MaskLo, MaskHi;
3512  if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3513  SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3514  } else {
3515  std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3516  }
3517 
3518  SDValue IndexHi, IndexLo;
3519  if (getTypeAction(Ops.Index.getValueType()) ==
3521  GetSplitVector(Ops.Index, IndexLo, IndexHi);
3522  else
3523  std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3524 
3525  SDValue Lo;
3526  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3527  N->getPointerInfo(), MachineMemOperand::MOStore,
3528  MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3529 
3530  if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3531  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3532  Lo =
3533  DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3534  MSC->getIndexType(), MSC->isTruncatingStore());
3535 
3536  // The order of the Scatter operation after split is well defined. The "Hi"
3537  // part comes after the "Lo". So these two operations should be chained one
3538  // after another.
3539  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3540  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3541  MMO, MSC->getIndexType(),
3542  MSC->isTruncatingStore());
3543  }
3544  auto *VPSC = cast<VPScatterSDNode>(N);
3545  SDValue EVLLo, EVLHi;
3546  std::tie(EVLLo, EVLHi) =
3547  DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3548 
3549  SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3550  Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3551  VPSC->getIndexType());
3552 
3553  // The order of the Scatter operation after split is well defined. The "Hi"
3554  // part comes after the "Lo". So these two operations should be chained one
3555  // after another.
3556  SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3557  return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3558  VPSC->getIndexType());
3559 }
3560 
3561 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3562  assert(N->isUnindexed() && "Indexed store of vector?");
3563  assert(OpNo == 1 && "Can only split the stored value");
3564  SDLoc DL(N);
3565 
3566  bool isTruncating = N->isTruncatingStore();
3567  SDValue Ch = N->getChain();
3568  SDValue Ptr = N->getBasePtr();
3569  EVT MemoryVT = N->getMemoryVT();
3570  Align Alignment = N->getOriginalAlign();
3571  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3572  AAMDNodes AAInfo = N->getAAInfo();
3573  SDValue Lo, Hi;
3574  GetSplitVector(N->getOperand(1), Lo, Hi);
3575 
3576  EVT LoMemVT, HiMemVT;
3577  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3578 
3579  // Scalarize if the split halves are not byte-sized.
3580  if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3581  return TLI.scalarizeVectorStore(N, DAG);
3582 
3583  if (isTruncating)
3584  Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3585  Alignment, MMOFlags, AAInfo);
3586  else
3587  Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3588  AAInfo);
3589 
3590  MachinePointerInfo MPI;
3591  IncrementPointer(N, LoMemVT, MPI, Ptr);
3592 
3593  if (isTruncating)
3594  Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3595  HiMemVT, Alignment, MMOFlags, AAInfo);
3596  else
3597  Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3598 
3599  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3600 }
3601 
3602 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3603  SDLoc DL(N);
3604 
3605  // The input operands all must have the same type, and we know the result
3606  // type is valid. Convert this to a buildvector which extracts all the
3607  // input elements.
3608  // TODO: If the input elements are power-two vectors, we could convert this to
3609  // a new CONCAT_VECTORS node with elements that are half-wide.
3611  EVT EltVT = N->getValueType(0).getVectorElementType();
3612  for (const SDValue &Op : N->op_values()) {
3613  for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3614  i != e; ++i) {
3615  Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3616  DAG.getVectorIdxConstant(i, DL)));
3617  }
3618  }
3619 
3620  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3621 }
3622 
3623 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3624  // The result type is legal, but the input type is illegal. If splitting
3625  // ends up with the result type of each half still being legal, just
3626  // do that. If, however, that would result in an illegal result type,
3627  // we can try to get more clever with power-two vectors. Specifically,
3628  // split the input type, but also widen the result element size, then
3629  // concatenate the halves and truncate again. For example, consider a target
3630  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3631  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3632  // %inlo = v4i32 extract_subvector %in, 0
3633  // %inhi = v4i32 extract_subvector %in, 4
3634  // %lo16 = v4i16 trunc v4i32 %inlo
3635  // %hi16 = v4i16 trunc v4i32 %inhi
3636  // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3637  // %res = v8i8 trunc v8i16 %in16
3638  //
3639  // Without this transform, the original truncate would end up being
3640  // scalarized, which is pretty much always a last resort.
3641  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3642  SDValue InVec = N->getOperand(OpNo);
3643  EVT InVT = InVec->getValueType(0);
3644  EVT OutVT = N->getValueType(0);
3645  ElementCount NumElements = OutVT.getVectorElementCount();
3646  bool IsFloat = OutVT.isFloatingPoint();
3647 
3648  unsigned InElementSize = InVT.getScalarSizeInBits();
3649  unsigned OutElementSize = OutVT.getScalarSizeInBits();
3650 
3651  // Determine the split output VT. If its legal we can just split dirctly.
3652  EVT LoOutVT, HiOutVT;
3653  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3654  assert(LoOutVT == HiOutVT && "Unequal split?");
3655 
3656  // If the input elements are only 1/2 the width of the result elements,
3657  // just use the normal splitting. Our trick only work if there's room
3658  // to split more than once.
3659  if (isTypeLegal(LoOutVT) ||
3660  InElementSize <= OutElementSize * 2)
3661  return SplitVecOp_UnaryOp(N);
3662  SDLoc DL(N);
3663 
3664  // Don't touch if this will be scalarized.
3665  EVT FinalVT = InVT;
3666  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3667  FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3668 
3669  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3670  return SplitVecOp_UnaryOp(N);
3671 
3672  // Get the split input vector.
3673  SDValue InLoVec, InHiVec;
3674  GetSplitVector(InVec, InLoVec, InHiVec);
3675 
3676  // Truncate them to 1/2 the element size.
3677  //
3678  // This assumes the number of elements is a power of two; any vector that
3679  // isn't should be widened, not split.
3680  EVT HalfElementVT = IsFloat ?
3681  EVT::getFloatingPointVT(InElementSize/2) :
3682  EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3683  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3684  NumElements.divideCoefficientBy(2));
3685 
3686  SDValue HalfLo;
3687  SDValue HalfHi;
3688  SDValue Chain;
3689  if (N->isStrictFPOpcode()) {
3690  HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3691  {N->getOperand(0), InLoVec});
3692  HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3693  {N->getOperand(0), InHiVec});
3694  // Legalize the chain result - switch anything that used the old chain to
3695  // use the new one.
3696  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3697  HalfHi.getValue(1));
3698  } else {
3699  HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3700  HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3701  }
3702 
3703  // Concatenate them to get the full intermediate truncation result.
3704  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3705  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3706  HalfHi);
3707  // Now finish up by truncating all the way down to the original result
3708  // type. This should normally be something that ends up being legal directly,
3709  // but in theory if a target has very wide vectors and an annoyingly
3710  // restricted set of legal types, this split can chain to build things up.
3711 
3712  if (N->isStrictFPOpcode()) {
3713  SDValue Res = DAG.getNode(
3714  ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3715  {Chain, InterVec,
3716  DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3717  // Relink the chain
3718  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3719  return Res;
3720  }
3721 
3722  return IsFloat
3723  ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3724  DAG.getTargetConstant(
3725  0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3726  : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3727 }
3728 
3729 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3730  assert(N->getValueType(0).isVector() &&
3731  N->getOperand(0).getValueType().isVector() &&
3732  "Operand types must be vectors");
3733  // The result has a legal vector type, but the input needs splitting.
3734  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3735  SDLoc DL(N);
3736  GetSplitVector(N->getOperand(0), Lo0, Hi0);
3737  GetSplitVector(N->getOperand(1), Lo1, Hi1);
3738  auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3739 
3740  LLVMContext &Context = *DAG.getContext();
3741  EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3742  EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3743 
3744  if (N->getOpcode() == ISD::SETCC) {
3745  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
3746  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
3747  } else {
3748  assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3749  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3750  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
3751  std::tie(EVLLo, EVLHi) =
3752  DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
3753  LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1,
3754  N->getOperand(2), MaskLo, EVLLo);
3755  HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1,
3756  N->getOperand(2), MaskHi, EVLHi);
3757  }
3758  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
3759 
3760  EVT OpVT = N->getOperand(0).getValueType();
3761  ISD::NodeType ExtendCode =
3763  return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
3764 }
3765 
3766 
3767 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3768  // The result has a legal vector type, but the input needs splitting.
3769  EVT ResVT = N->getValueType(0);
3770  SDValue Lo, Hi;
3771  SDLoc DL(N);
3772  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3773  EVT InVT = Lo.getValueType();
3774 
3775  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3776  InVT.getVectorElementCount());
3777 
3778  if (N->isStrictFPOpcode()) {
3779  Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3780  { N->getOperand(0), Lo, N->getOperand(2) });
3781  Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3782  { N->getOperand(0), Hi, N->getOperand(2) });
3783  // Legalize the chain result - switch anything that used the old chain to
3784  // use the new one.
3785  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3786  Lo.getValue(1), Hi.getValue(1));
3787  ReplaceValueWith(SDValue(N, 1), NewChain);
3788  } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
3789  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3790  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3791  std::tie(EVLLo, EVLHi) =
3792  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL);
3793  Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo);
3794  Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi);
3795  } else {
3796  Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
3797  Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
3798  }
3799 
3800  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
3801 }
3802 
3803 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
3804  // The result (and the first input) has a legal vector type, but the second
3805  // input needs splitting.
3806 
3807  SDLoc DL(N);
3808 
3809  EVT LHSLoVT, LHSHiVT;
3810  std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
3811 
3812  if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
3813  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
3814 
3815  SDValue LHSLo, LHSHi;
3816  std::tie(LHSLo, LHSHi) =
3817  DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT);
3818 
3819  SDValue RHSLo, RHSHi;
3820  std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL);
3821 
3822  SDValue Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLoVT, LHSLo, RHSLo);
3823  SDValue Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHiVT, LHSHi, RHSHi);
3824 
3825  return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
3826 }
3827 
3828 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
3829  EVT ResVT = N->getValueType(0);
3830  SDValue Lo, Hi;
3831  SDLoc dl(N);
3832  GetSplitVector(N->getOperand(0), Lo, Hi);
3833  EVT InVT = Lo.getValueType();
3834 
3835  EVT NewResVT =
3836  EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3837  InVT.getVectorElementCount());
3838 
3839  Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3840  Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3841 
3842  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3843 }
3844 
3845 //===----------------------------------------------------------------------===//
3846 // Result Vector Widening
3847 //===----------------------------------------------------------------------===//
3848 
3849 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3850  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3851  dbgs() << "\n");
3852 
3853  // See if the target wants to custom widen this node.
3854  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3855  return;
3856 
3857  SDValue Res = SDValue();
3858 
3859  auto unrollExpandedOp = [&]() {
3860  // We're going to widen this vector op to a legal type by padding with undef
3861  // elements. If the wide vector op is eventually going to be expanded to
3862  // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3863  // libcalls on the undef elements.
3864  EVT VT = N->getValueType(0);
3865  EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3866  if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3867  TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3868  Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3869  return true;
3870  }
3871  return false;
3872  };
3873 
3874  switch (N->getOpcode()) {
3875  default:
3876 #ifndef NDEBUG
3877  dbgs() << "WidenVectorResult #" << ResNo << ": ";
3878  N->dump(&DAG);
3879  dbgs() << "\n";
3880 #endif
3881  llvm_unreachable("Do not know how to widen the result of this operator!");
3882 
3883  case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3884  case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3885  case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3886  case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3887  case ISD::INSERT_SUBVECTOR:
3888  Res = WidenVecRes_INSERT_SUBVECTOR(N);
3889  break;
3890  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3891  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3892  case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3893  case ISD::STEP_VECTOR:
3894  case ISD::SPLAT_VECTOR:
3895  case ISD::SCALAR_TO_VECTOR:
3896  Res = WidenVecRes_ScalarOp(N);
3897  break;
3898  case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3899  case ISD::VSELECT:
3900  case ISD::SELECT:
3901  case ISD::VP_SELECT:
3902  case ISD::VP_MERGE:
3903  Res = WidenVecRes_Select(N);
3904  break;
3905  case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3906  case ISD::VP_SETCC:
3907  case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3908  case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3909  case ISD::VECTOR_SHUFFLE:
3910  Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3911  break;
3912  case ISD::VP_LOAD:
3913  Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
3914  break;
3915  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
3916  Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N));
3917  break;
3918  case ISD::MLOAD:
3919  Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3920  break;
3921  case ISD::MGATHER:
3922  Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3923  break;
3924  case ISD::VP_GATHER:
3925  Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
3926  break;
3927  case ISD::VECTOR_REVERSE:
3928  Res = WidenVecRes_VECTOR_REVERSE(N);
3929  break;
3930 
3931  case ISD::ADD: case ISD::VP_ADD:
3932  case ISD::AND: case ISD::VP_AND:
3933  case ISD::MUL: case ISD::VP_MUL:
3934  case ISD::MULHS:
3935  case ISD::MULHU:
3936  case ISD::OR: case ISD::VP_OR:
3937  case ISD::SUB: case ISD::VP_SUB:
3938  case ISD::XOR: case ISD::VP_XOR:
3939  case ISD::SHL: case ISD::VP_SHL:
3940  case ISD::SRA: case ISD::VP_ASHR:
3941  case ISD::SRL: case ISD::VP_LSHR:
3942  case ISD::FMINNUM: case ISD::VP_FMINNUM:
3943  case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
3944  case ISD::FMINIMUM:
3945  case ISD::FMAXIMUM:
3946  case ISD::SMIN: case ISD::VP_SMIN:
3947  case ISD::SMAX: case ISD::VP_SMAX:
3948  case ISD::UMIN: case ISD::VP_UMIN:
3949  case ISD::UMAX: case ISD::VP_UMAX:
3950  case ISD::UADDSAT:
3951  case ISD::SADDSAT:
3952  case ISD::USUBSAT:
3953  case ISD::SSUBSAT:
3954  case ISD::SSHLSAT:
3955  case ISD::USHLSAT:
3956  case ISD::ROTL:
3957  case ISD::ROTR:
3958  case ISD::AVGFLOORS:
3959  case ISD::AVGFLOORU:
3960  case ISD::AVGCEILS:
3961  case ISD::AVGCEILU:
3962  // Vector-predicated binary op widening. Note that -- unlike the
3963  // unpredicated versions -- we don't have to worry about trapping on
3964  // operations like UDIV, FADD, etc., as we pass on the original vector
3965  // length parameter. This means the widened elements containing garbage
3966  // aren't active.
3967  case ISD::VP_SDIV:
3968  case ISD::VP_UDIV:
3969  case ISD::VP_SREM:
3970  case ISD::VP_UREM:
3971  case ISD::VP_FADD:
3972  case ISD::VP_FSUB:
3973  case ISD::VP_FMUL:
3974  case ISD::VP_FDIV:
3975  case ISD::VP_FREM:
3976  case ISD::VP_FCOPYSIGN:
3977  Res = WidenVecRes_Binary(N);
3978  break;
3979 
3980  case ISD::FPOW:
3981  case ISD::FREM:
3982  if (unrollExpandedOp())
3983  break;
3984  // If the target has custom/legal support for the scalar FP intrinsic ops
3985  // (they are probably not destined to become libcalls), then widen those
3986  // like any other binary ops.
3987  [[fallthrough]];
3988 
3989  case ISD::FADD:
3990  case ISD::FMUL:
3991  case ISD::FSUB:
3992  case ISD::FDIV:
3993  case ISD::SDIV:
3994  case ISD::UDIV:
3995  case ISD::SREM:
3996  case ISD::UREM:
3997  Res = WidenVecRes_BinaryCanTrap(N);
3998  break;
3999 
4000  case ISD::SMULFIX:
4001  case ISD::SMULFIXSAT:
4002  case ISD::UMULFIX:
4003  case ISD::UMULFIXSAT:
4004  // These are binary operations, but with an extra operand that shouldn't
4005  // be widened (the scale).
4006  Res = WidenVecRes_BinaryWithExtraScalarOp(N);
4007  break;
4008 
4009 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4010  case ISD::STRICT_##DAGN:
4011 #include "llvm/IR/ConstrainedOps.def"
4012  Res = WidenVecRes_StrictFP(N);
4013  break;
4014 
4015  case ISD::UADDO:
4016  case ISD::SADDO:
4017  case ISD::USUBO:
4018  case ISD::SSUBO:
4019  case ISD::UMULO:
4020  case ISD::SMULO:
4021  Res = WidenVecRes_OverflowOp(N, ResNo);
4022  break;
4023 
4024  case ISD::FCOPYSIGN:
4025  Res = WidenVecRes_FCOPYSIGN(N);
4026  break;
4027 
4028  case ISD::IS_FPCLASS:
4029  Res = WidenVecRes_IS_FPCLASS(N);
4030  break;
4031 
4032  case ISD::FPOWI:
4033  Res = WidenVecRes_POWI(N);
4034  break;
4035 
4039  Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
4040  break;
4041 
4042  case ISD::ANY_EXTEND:
4043  case ISD::FP_EXTEND:
4044  case ISD::VP_FP_EXTEND:
4045  case ISD::FP_ROUND:
4046  case ISD::VP_FP_ROUND:
4047  case ISD::FP_TO_SINT:
4048  case ISD::VP_FP_TO_SINT:
4049  case ISD::FP_TO_UINT:
4050  case ISD::VP_FP_TO_UINT:
4051  case ISD::SIGN_EXTEND:
4052  case ISD::VP_SIGN_EXTEND:
4053  case ISD::SINT_TO_FP:
4054  case ISD::VP_SINT_TO_FP:
4055  case ISD::VP_TRUNCATE:
4056  case ISD::TRUNCATE:
4057  case ISD::UINT_TO_FP:
4058  case ISD::VP_UINT_TO_FP:
4059  case ISD::ZERO_EXTEND:
4060  case ISD::VP_ZERO_EXTEND:
4061  Res = WidenVecRes_Convert(N);
4062  break;
4063 
4064  case ISD::FP_TO_SINT_SAT:
4065  case ISD::FP_TO_UINT_SAT:
4066  Res = WidenVecRes_FP_TO_XINT_SAT(N);
4067  break;
4068 
4069  case ISD::FABS:
4070  case ISD::FCEIL:
4071  case ISD::FCOS:
4072  case ISD::FEXP:
4073  case ISD::FEXP2:
4074  case ISD::FFLOOR:
4075  case ISD::FLOG:
4076  case ISD::FLOG10:
4077  case ISD::FLOG2:
4078  case ISD::FNEARBYINT:
4079  case ISD::FRINT:
4080  case ISD::FROUND:
4081  case ISD::FROUNDEVEN:
4082  case ISD::FSIN:
4083  case ISD::FSQRT:
4084  case ISD::FTRUNC:
4085  if (unrollExpandedOp())
4086  break;
4087  // If the target has custom/legal support for the scalar FP intrinsic ops
4088  // (they are probably not destined to become libcalls), then widen those
4089  // like any other unary ops.
4090  [[fallthrough]];
4091 
4092  case ISD::ABS:
4093  case ISD::BITREVERSE:
4094  case ISD::BSWAP:
4095  case ISD::VP_BSWAP:
4096  case ISD::CTLZ:
4097  case ISD::CTLZ_ZERO_UNDEF:
4098  case ISD::CTPOP:
4099  case ISD::CTTZ:
4100  case ISD::CTTZ_ZERO_UNDEF:
4101  case ISD::FNEG: case ISD::VP_FNEG:
4102  case ISD::VP_FABS:
4103  case ISD::VP_SQRT:
4104  case ISD::VP_FCEIL:
4105  case ISD::VP_FFLOOR:
4106  case ISD::VP_FRINT:
4107  case ISD::VP_FNEARBYINT:
4108  case ISD::VP_FROUND:
4109  case ISD::VP_FROUNDEVEN:
4110  case ISD::VP_FROUNDTOZERO:
4111  case ISD::FREEZE:
4112  case ISD::ARITH_FENCE:
4113  case ISD::FCANONICALIZE:
4114  Res = WidenVecRes_Unary(N);
4115  break;
4116  case ISD::FMA: case ISD::VP_FMA:
4117  case ISD::FSHL:
4118  case ISD::VP_FSHL:
4119  case ISD::FSHR:
4120  case ISD::VP_FSHR:
4121  Res = WidenVecRes_Ternary(N);
4122  break;
4123  }
4124 
4125  // If Res is null, the sub-method took care of registering the result.
4126  if (Res.getNode())
4127  SetWidenedVector(SDValue(N, ResNo), Res);
4128 }
4129 
4130 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
4131  // Ternary op widening.
4132  SDLoc dl(N);
4133  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4134  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4135  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4136  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
4137  if (N->getNumOperands() == 3)
4138  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
4139 
4140  assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
4141  assert(N->isVPOpcode() && "Expected VP opcode");
4142 
4143  SDValue Mask =
4144  GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount());
4145  return DAG.getNode(N->getOpcode(), dl, WidenVT,
4146  {InOp1, InOp2, InOp3, Mask, N->getOperand(4)});
4147 }
4148 
4149 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
4150  // Binary op widening.
4151  SDLoc dl(N);
4152  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4153  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4154  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4155  if (N->getNumOperands() == 2)
4156  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
4157  N->getFlags());
4158 
4159  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
4160  assert(N->isVPOpcode() && "Expected VP opcode");
4161 
4162  SDValue Mask =
4163  GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount());
4164  return DAG.getNode(N->getOpcode(), dl, WidenVT,
4165  {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
4166 }
4167 
4168 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
4169  // Binary op widening, but with an extra operand that shouldn't be widened.
4170  SDLoc dl(N);
4171  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4172  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4173  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4174  SDValue InOp3 = N->getOperand(2);
4175  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
4176  N->getFlags());
4177 }
4178 
4179 // Given a vector of operations that have been broken up to widen, see
4180 // if we can collect them together into the next widest legal VT. This
4181 // implementation is trap-safe.
4183  SmallVectorImpl<SDValue> &ConcatOps,
4184  unsigned ConcatEnd, EVT VT, EVT MaxVT,
4185  EVT WidenVT) {
4186  // Check to see if we have a single operation with the widen type.
4187  if (ConcatEnd == 1) {
4188  VT = ConcatOps[0].getValueType();
4189  if (VT == WidenVT)
4190  return ConcatOps[0];
4191  }
4192 
4193  SDLoc dl(ConcatOps[0]);
4194  EVT WidenEltVT = WidenVT.getVectorElementType();
4195 
4196  // while (Some element of ConcatOps is not of type MaxVT) {
4197  // From the end of ConcatOps, collect elements of the same type and put
4198  // them into an op of the next larger supported type
4199  // }
4200  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
4201  int Idx = ConcatEnd - 1;
4202  VT = ConcatOps[Idx--].getValueType();
4203  while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
4204  Idx--;
4205 
4206  int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
4207  EVT NextVT;
4208  do {
4209  NextSize *= 2;
4210  NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
4211  } while (!TLI.isTypeLegal(NextVT));
4212 
4213  if (!VT.isVector()) {
4214  // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
4215  SDValue VecOp = DAG.getUNDEF(NextVT);
4216  unsigned NumToInsert = ConcatEnd - Idx - 1;
4217  for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
4218  VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
4219  ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
4220  }
4221  ConcatOps[Idx+1] = VecOp;
4222  ConcatEnd = Idx + 2;
4223  } else {
4224  // Vector type, create a CONCAT_VECTORS of type NextVT
4225  SDValue undefVec = DAG.getUNDEF(VT);
4226  unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
4227  SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
4228  unsigned RealVals = ConcatEnd - Idx - 1;
4229  unsigned SubConcatEnd = 0;
4230  unsigned SubConcatIdx = Idx + 1;
4231  while (SubConcatEnd < RealVals)
4232  SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
4233  while (SubConcatEnd < OpsToConcat)
4234  SubConcatOps[SubConcatEnd++] = undefVec;
4235  ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
4236  NextVT, SubConcatOps);
4237  ConcatEnd = SubConcatIdx + 1;
4238  }
4239  }
4240 
4241  // Check to see if we have a single operation with the widen type.
4242  if (ConcatEnd == 1) {
4243  VT = ConcatOps[0].getValueType();
4244  if (VT == WidenVT)
4245  return ConcatOps[0];
4246  }
4247 
4248  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
4249  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
4250  if (NumOps != ConcatEnd ) {
4251  SDValue UndefVal = DAG.getUNDEF(MaxVT);
4252  for (unsigned j = ConcatEnd; j < NumOps; ++j)
4253  ConcatOps[j] = UndefVal;
4254  }
4255  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4256  makeArrayRef(ConcatOps.data(), NumOps));
4257 }
4258 
4259 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
4260  // Binary op widening for operations that can trap.
4261  unsigned Opcode = N->getOpcode();
4262  SDLoc dl(N);
4263  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4264  EVT WidenEltVT = WidenVT.getVectorElementType();
4265  EVT VT = WidenVT;
4266  unsigned NumElts = VT.getVectorMinNumElements();
4267  const SDNodeFlags Flags = N->getFlags();
4268  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4269  NumElts = NumElts / 2;
4270  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4271  }
4272 
4273  if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
4274  // Operation doesn't trap so just widen as normal.
4275  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4276  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4277  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
4278  }
4279 
4280  // FIXME: Improve support for scalable vectors.
4281  assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
4282 
4283  // No legal vector version so unroll the vector operation and then widen.
4284  if (NumElts == 1)
4285  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4286 
4287  // Since the operation can trap, apply operation on the original vector.
4288  EVT MaxVT = VT;
4289  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4290  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4291  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4292 
4293  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4294  unsigned ConcatEnd = 0; // Current ConcatOps index.
4295  int Idx = 0; // Current Idx into input vectors.
4296 
4297  // NumElts := greatest legal vector size (at most WidenVT)
4298  // while (orig. vector has unhandled elements) {
4299  // take munches of size NumElts from the beginning and add to ConcatOps
4300  // NumElts := next smaller supported vector size or 1
4301  // }
4302  while (CurNumElts != 0) {
4303  while (CurNumElts >= NumElts) {
4304  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
4305  DAG.getVectorIdxConstant(Idx, dl));
4306  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
4307  DAG.getVectorIdxConstant(Idx, dl));
4308  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
4309  Idx += NumElts;
4310  CurNumElts -= NumElts;
4311  }
4312  do {
4313  NumElts = NumElts / 2;
4314  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4315  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4316 
4317  if (NumElts == 1) {
4318  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4319  SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4320  InOp1, DAG.getVectorIdxConstant(Idx, dl));
4321  SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4322  InOp2, DAG.getVectorIdxConstant(Idx, dl));
4323  ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
4324  EOp1, EOp2, Flags);
4325  }
4326  CurNumElts = 0;
4327  }
4328  }
4329 
4330  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4331 }
4332 
4333 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
4334  switch (N->getOpcode()) {
4335  case ISD::STRICT_FSETCC:
4336  case ISD::STRICT_FSETCCS:
4337  return WidenVecRes_STRICT_FSETCC(N);
4338  case ISD::STRICT_FP_EXTEND:
4339  case ISD::STRICT_FP_ROUND:
4344  return WidenVecRes_Convert_StrictFP(N);
4345  default:
4346  break;
4347  }
4348 
4349  // StrictFP op widening for operations that can trap.
4350  unsigned NumOpers = N->getNumOperands();
4351  unsigned Opcode = N->getOpcode();
4352  SDLoc dl(N);
4353  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4354  EVT WidenEltVT = WidenVT.getVectorElementType();
4355  EVT VT = WidenVT;
4356  unsigned NumElts = VT.getVectorNumElements();
4357  while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4358  NumElts = NumElts / 2;
4359  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4360  }
4361 
4362  // No legal vector version so unroll the vector operation and then widen.
4363  if (NumElts == 1)
4364  return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
4365 
4366  // Since the operation can trap, apply operation on the original vector.
4367  EVT MaxVT = VT;
4369  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4370 
4371  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4372  SmallVector<SDValue, 16> Chains;
4373  unsigned ConcatEnd = 0; // Current ConcatOps index.
4374  int Idx = 0; // Current Idx into input vectors.
4375 
4376  // The Chain is the first operand.
4377  InOps.push_back(N->getOperand(0));
4378 
4379  // Now process the remaining operands.
4380  for (unsigned i = 1; i < NumOpers; ++i) {
4381  SDValue Oper = N->getOperand(i);
4382 
4383  if (Oper.getValueType().isVector()) {
4384  assert(Oper.getValueType() == N->getValueType(0) &&
4385  "Invalid operand type to widen!");
4386  Oper = GetWidenedVector(Oper);
4387  }
4388 
4389  InOps.push_back(Oper);
4390  }
4391 
4392  // NumElts := greatest legal vector size (at most WidenVT)
4393  // while (orig. vector has unhandled elements) {
4394  // take munches of size NumElts from the beginning and add to ConcatOps
4395  // NumElts := next smaller supported vector size or 1
4396  // }
4397  while (CurNumElts != 0) {
4398  while (CurNumElts >= NumElts) {
4400 
4401  for (unsigned i = 0; i < NumOpers; ++i) {
4402  SDValue Op = InOps[i];
4403 
4404  if (Op.getValueType().isVector())
4405  Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
4406  DAG.getVectorIdxConstant(Idx, dl));
4407 
4408  EOps.push_back(Op);
4409  }
4410 
4411  EVT OperVT[] = {VT, MVT::Other};
4412  SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
4413  ConcatOps[ConcatEnd++] = Oper;
4414  Chains.push_back(Oper.getValue(1));
4415  Idx += NumElts;
4416  CurNumElts -= NumElts;
4417  }
4418  do {
4419  NumElts = NumElts / 2;
4420  VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4421  } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4422 
4423  if (NumElts == 1) {
4424  for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4426 
4427  for (unsigned i = 0; i < NumOpers; ++i) {
4428  SDValue Op = InOps[i];
4429 
4430  if (Op.getValueType().isVector())
4431  Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
4432  DAG.getVectorIdxConstant(Idx, dl));
4433 
4434  EOps.push_back(Op);
4435  }
4436 
4437  EVT WidenVT[] = {WidenEltVT, MVT::Other};
4438  SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
4439  ConcatOps[ConcatEnd++] = Oper;
4440  Chains.push_back(Oper.getValue(1));
4441  }
4442  CurNumElts = 0;
4443  }
4444  }
4445 
4446  // Build a factor node to remember all the Ops that have been created.
4447  SDValue NewChain;
4448  if (Chains.size() == 1)
4449  NewChain = Chains[0];
4450  else
4451  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4452  ReplaceValueWith(SDValue(N, 1), NewChain);
4453 
4454  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4455 }
4456 
4457 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
4458  SDLoc DL(N);
4459  EVT ResVT = N->getValueType(0);
4460  EVT OvVT = N->getValueType(1);
4461  EVT WideResVT, WideOvVT;
4462  SDValue WideLHS, WideRHS;
4463 
4464  // TODO: This might result in a widen/split loop.
4465  if (ResNo == 0) {
4466  WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
4467  WideOvVT = EVT::getVectorVT(
4468  *DAG.getContext(), OvVT.getVectorElementType(),
4469  WideResVT.getVectorNumElements());
4470 
4471  WideLHS = GetWidenedVector(N->getOperand(0));
4472  WideRHS = GetWidenedVector(N->getOperand(1));
4473  } else {
4474  WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
4475  WideResVT = EVT::getVectorVT(
4476  *DAG.getContext(), ResVT.getVectorElementType(),
4477  WideOvVT.getVectorNumElements());
4478 
4479  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4480  WideLHS = DAG.getNode(
4481  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4482  N->getOperand(0), Zero);
4483  WideRHS = DAG.getNode(
4484  ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4485  N->getOperand(1), Zero);
4486  }
4487 
4488  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
4489  SDNode *WideNode = DAG.getNode(
4490  N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
4491 
4492  // Replace the other vector result not being explicitly widened here.
4493  unsigned OtherNo = 1 - ResNo;
4494  EVT OtherVT = N->getValueType(OtherNo);
4495  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
4496  SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
4497  } else {
4498  SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4499  SDValue OtherVal = DAG.getNode(
4500  ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
4501  ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
4502  }
4503 
4504  return SDValue(WideNode, ResNo);
4505 }
4506 
4507 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
4508  LLVMContext &Ctx = *DAG.getContext();
4509  SDValue InOp = N->getOperand(0);
4510  SDLoc DL(N);
4511 
4512  EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
4513  ElementCount WidenEC = WidenVT.getVectorElementCount();
4514 
4515  EVT InVT = InOp.getValueType();
4516 
4517  unsigned Opcode = N->getOpcode();
4518  const SDNodeFlags Flags = N->getFlags();
4519 
4520  // Handle the case of ZERO_EXTEND where the promoted InVT element size does
4521  // not equal that of WidenVT.
4522  if (N->getOpcode() == ISD::ZERO_EXTEND &&
4523  getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
4524  TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
4525  WidenVT.getScalarSizeInBits()) {
4526  InOp = ZExtPromotedInteger(InOp);
4527  InVT = InOp.getValueType();
4528  if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
4529  Opcode = ISD::TRUNCATE;
4530  }
4531 
4532  EVT InEltVT = InVT.getVectorElementType();
4533  EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
4534  ElementCount InVTEC = InVT.getVectorElementCount();
4535 
4536  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4537  InOp = GetWidenedVector(N->getOperand(0));
4538  InVT = InOp.getValueType();
4539  InVTEC = InVT.getVectorElementCount();
4540  if (InVTEC == WidenEC) {
4541  if (N->getNumOperands() == 1)
4542  return DAG.getNode(Opcode, DL, WidenVT, InOp);
4543  if (N->getNumOperands() == 3) {
4544  assert(N->isVPOpcode() && "Expected VP opcode");
4545  SDValue Mask =
4546  GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount());
4547  return DAG.getNode(Opcode, DL, WidenVT, InOp, Mask, N->getOperand(2));
4548  }
4549  return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
4550  }
4551  if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
4552  // If both input and result vector types are of same width, extend
4553  // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
4554  // accepts fewer elements in the result than in the input.
4555  if (Opcode == ISD::ANY_EXTEND)
4556  return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4557  if (Opcode == ISD::SIGN_EXTEND)
4558  return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4559  if (Opcode == ISD::ZERO_EXTEND)
4560  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4561  }
4562  }
4563 
4564  if (TLI.isTypeLegal(InWidenVT)) {
4565  // Because the result and the input are different vector types, widening
4566  // the result could create a legal type but widening the input might make
4567  // it an illegal type that might lead to repeatedly splitting the input
4568  // and then widening it. To avoid this, we widen the input only if
4569  // it results in a legal type.
4570  if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
4571  // Widen the input and call convert on the widened input vector.
4572  unsigned NumConcat =
4573  WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
4574  SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
4575  Ops[0] = InOp;
4576  SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
4577  if (N->getNumOperands() == 1)
4578  return DAG.getNode(Opcode, DL, WidenVT, InVec);
4579  return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
4580  }
4581 
4582  if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
4583  SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
4584  DAG.getVectorIdxConstant(0, DL));
4585  // Extract the input and convert the shorten input vector.
4586  if (N->getNumOperands() == 1)
4587  return DAG.getNode(Opcode, DL, WidenVT, InVal);
4588  return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
4589  }
4590  }
4591 
4592  // Otherwise unroll into some nasty scalar code and rebuild the vector.
4593  EVT EltVT = WidenVT.getVectorElementType();
4594  SmallVector<SDValue, 16> Ops(WidenEC.