LLVM 17.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
26#include "llvm/IR/DataLayout.h"
30#include <numeric>
31
32using namespace llvm;
33
34#define DEBUG_TYPE "legalize-types"
35
36//===----------------------------------------------------------------------===//
37// Result Vector Scalarization: <1 x ty> -> ty.
38//===----------------------------------------------------------------------===//
39
40void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
41 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
42 dbgs() << "\n");
43 SDValue R = SDValue();
44
45 switch (N->getOpcode()) {
46 default:
47#ifndef NDEBUG
48 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
49 N->dump(&DAG);
50 dbgs() << "\n";
51#endif
52 report_fatal_error("Do not know how to scalarize the result of this "
53 "operator!\n");
54
55 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
56 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
57 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
58 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
59 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
60 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
61 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
62 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
63 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
64 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
65 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
66 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
67 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
68 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
69 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
70 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
75 R = ScalarizeVecRes_VecInregOp(N);
76 break;
77 case ISD::ABS:
78 case ISD::ANY_EXTEND:
79 case ISD::BITREVERSE:
80 case ISD::BSWAP:
81 case ISD::CTLZ:
83 case ISD::CTPOP:
84 case ISD::CTTZ:
86 case ISD::FABS:
87 case ISD::FCEIL:
88 case ISD::FCOS:
89 case ISD::FEXP:
90 case ISD::FEXP2:
91 case ISD::FFLOOR:
92 case ISD::FLOG:
93 case ISD::FLOG10:
94 case ISD::FLOG2:
95 case ISD::FNEARBYINT:
96 case ISD::FNEG:
97 case ISD::FREEZE:
99 case ISD::FP_EXTEND:
100 case ISD::FP_TO_SINT:
101 case ISD::FP_TO_UINT:
102 case ISD::FRINT:
103 case ISD::FROUND:
104 case ISD::FROUNDEVEN:
105 case ISD::FSIN:
106 case ISD::FSQRT:
107 case ISD::FTRUNC:
108 case ISD::SIGN_EXTEND:
109 case ISD::SINT_TO_FP:
110 case ISD::TRUNCATE:
111 case ISD::UINT_TO_FP:
112 case ISD::ZERO_EXTEND:
114 R = ScalarizeVecRes_UnaryOp(N);
115 break;
116
117 case ISD::ADD:
118 case ISD::AND:
119 case ISD::FADD:
120 case ISD::FCOPYSIGN:
121 case ISD::FDIV:
122 case ISD::FMUL:
123 case ISD::FMINNUM:
124 case ISD::FMAXNUM:
127 case ISD::FMINIMUM:
128 case ISD::FMAXIMUM:
129 case ISD::SMIN:
130 case ISD::SMAX:
131 case ISD::UMIN:
132 case ISD::UMAX:
133
134 case ISD::SADDSAT:
135 case ISD::UADDSAT:
136 case ISD::SSUBSAT:
137 case ISD::USUBSAT:
138 case ISD::SSHLSAT:
139 case ISD::USHLSAT:
140
141 case ISD::FPOW:
142 case ISD::FREM:
143 case ISD::FSUB:
144 case ISD::MUL:
145 case ISD::OR:
146 case ISD::SDIV:
147 case ISD::SREM:
148 case ISD::SUB:
149 case ISD::UDIV:
150 case ISD::UREM:
151 case ISD::XOR:
152 case ISD::SHL:
153 case ISD::SRA:
154 case ISD::SRL:
155 case ISD::ROTL:
156 case ISD::ROTR:
157 R = ScalarizeVecRes_BinOp(N);
158 break;
159 case ISD::FMA:
160 case ISD::FSHL:
161 case ISD::FSHR:
162 R = ScalarizeVecRes_TernaryOp(N);
163 break;
164
165#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
166 case ISD::STRICT_##DAGN:
167#include "llvm/IR/ConstrainedOps.def"
168 R = ScalarizeVecRes_StrictFPOp(N);
169 break;
170
173 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
174 break;
175
176 case ISD::UADDO:
177 case ISD::SADDO:
178 case ISD::USUBO:
179 case ISD::SSUBO:
180 case ISD::UMULO:
181 case ISD::SMULO:
182 R = ScalarizeVecRes_OverflowOp(N, ResNo);
183 break;
184 case ISD::SMULFIX:
185 case ISD::SMULFIXSAT:
186 case ISD::UMULFIX:
187 case ISD::UMULFIXSAT:
188 case ISD::SDIVFIX:
189 case ISD::SDIVFIXSAT:
190 case ISD::UDIVFIX:
191 case ISD::UDIVFIXSAT:
192 R = ScalarizeVecRes_FIX(N);
193 break;
194 }
195
196 // If R is null, the sub-method took care of registering the result.
197 if (R.getNode())
198 SetScalarizedVector(SDValue(N, ResNo), R);
199}
200
201SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
202 SDValue LHS = GetScalarizedVector(N->getOperand(0));
203 SDValue RHS = GetScalarizedVector(N->getOperand(1));
204 return DAG.getNode(N->getOpcode(), SDLoc(N),
205 LHS.getValueType(), LHS, RHS, N->getFlags());
206}
207
208SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
209 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
210 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
211 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
212 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
213 Op2, N->getFlags());
214}
215
216SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
217 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
218 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
219 SDValue Op2 = N->getOperand(2);
220 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
221 Op2, N->getFlags());
222}
223
224SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
225 EVT VT = N->getValueType(0).getVectorElementType();
226 unsigned NumOpers = N->getNumOperands();
227 SDValue Chain = N->getOperand(0);
228 EVT ValueVTs[] = {VT, MVT::Other};
229 SDLoc dl(N);
230
231 SmallVector<SDValue, 4> Opers(NumOpers);
232
233 // The Chain is the first operand.
234 Opers[0] = Chain;
235
236 // Now process the remaining operands.
237 for (unsigned i = 1; i < NumOpers; ++i) {
238 SDValue Oper = N->getOperand(i);
239 EVT OperVT = Oper.getValueType();
240
241 if (OperVT.isVector()) {
242 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
243 Oper = GetScalarizedVector(Oper);
244 else
245 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
246 OperVT.getVectorElementType(), Oper,
247 DAG.getVectorIdxConstant(0, dl));
248 }
249
250 Opers[i] = Oper;
251 }
252
253 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
254 Opers, N->getFlags());
255
256 // Legalize the chain result - switch anything that used the old chain to
257 // use the new one.
258 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
259 return Result;
260}
261
262SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
263 unsigned ResNo) {
264 SDLoc DL(N);
265 EVT ResVT = N->getValueType(0);
266 EVT OvVT = N->getValueType(1);
267
268 SDValue ScalarLHS, ScalarRHS;
269 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
270 ScalarLHS = GetScalarizedVector(N->getOperand(0));
271 ScalarRHS = GetScalarizedVector(N->getOperand(1));
272 } else {
273 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
274 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
275 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
276 ScalarLHS = ElemsLHS[0];
277 ScalarRHS = ElemsRHS[0];
278 }
279
280 SDVTList ScalarVTs = DAG.getVTList(
282 SDNode *ScalarNode = DAG.getNode(
283 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
284 ScalarNode->setFlags(N->getFlags());
285
286 // Replace the other vector result not being explicitly scalarized here.
287 unsigned OtherNo = 1 - ResNo;
288 EVT OtherVT = N->getValueType(OtherNo);
289 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
290 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
291 } else {
292 SDValue OtherVal = DAG.getNode(
293 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
294 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
295 }
296
297 return SDValue(ScalarNode, ResNo);
298}
299
300SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
301 unsigned ResNo) {
302 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
303 return GetScalarizedVector(Op);
304}
305
306SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
307 SDValue Op = N->getOperand(0);
308 if (Op.getValueType().isVector()
309 && Op.getValueType().getVectorNumElements() == 1
310 && !isSimpleLegalType(Op.getValueType()))
311 Op = GetScalarizedVector(Op);
312 EVT NewVT = N->getValueType(0).getVectorElementType();
313 return DAG.getNode(ISD::BITCAST, SDLoc(N),
314 NewVT, Op);
315}
316
317SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
318 EVT EltVT = N->getValueType(0).getVectorElementType();
319 SDValue InOp = N->getOperand(0);
320 // The BUILD_VECTOR operands may be of wider element types and
321 // we may need to truncate them back to the requested return type.
322 if (EltVT.isInteger())
323 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
324 return InOp;
325}
326
327SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
329 N->getValueType(0).getVectorElementType(),
330 N->getOperand(0), N->getOperand(1));
331}
332
333SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
334 SDLoc DL(N);
335 SDValue Op = N->getOperand(0);
336 EVT OpVT = Op.getValueType();
337 // The result needs scalarizing, but it's not a given that the source does.
338 // See similar logic in ScalarizeVecRes_UnaryOp.
339 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
340 Op = GetScalarizedVector(Op);
341 } else {
342 EVT VT = OpVT.getVectorElementType();
343 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
344 DAG.getVectorIdxConstant(0, DL));
345 }
346 return DAG.getNode(ISD::FP_ROUND, DL,
347 N->getValueType(0).getVectorElementType(), Op,
348 N->getOperand(1));
349}
350
351SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
352 SDValue Op = GetScalarizedVector(N->getOperand(0));
353 return DAG.getNode(ISD::FPOWI, SDLoc(N),
354 Op.getValueType(), Op, N->getOperand(1));
355}
356
357SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
358 // The value to insert may have a wider type than the vector element type,
359 // so be sure to truncate it to the element type if necessary.
360 SDValue Op = N->getOperand(1);
361 EVT EltVT = N->getValueType(0).getVectorElementType();
362 if (Op.getValueType() != EltVT)
363 // FIXME: Can this happen for floating point types?
364 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
365 return Op;
366}
367
368SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
369 assert(N->isUnindexed() && "Indexed vector load?");
370
371 SDValue Result = DAG.getLoad(
372 ISD::UNINDEXED, N->getExtensionType(),
373 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
374 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
375 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
376 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
377
378 // Legalize the chain result - switch anything that used the old chain to
379 // use the new one.
380 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
381 return Result;
382}
383
384SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
385 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
386 EVT DestVT = N->getValueType(0).getVectorElementType();
387 SDValue Op = N->getOperand(0);
388 EVT OpVT = Op.getValueType();
389 SDLoc DL(N);
390 // The result needs scalarizing, but it's not a given that the source does.
391 // This is a workaround for targets where it's impossible to scalarize the
392 // result of a conversion, because the source type is legal.
393 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
394 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
395 // legal and was not scalarized.
396 // See the similar logic in ScalarizeVecRes_SETCC
397 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
398 Op = GetScalarizedVector(Op);
399 } else {
400 EVT VT = OpVT.getVectorElementType();
401 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
402 DAG.getVectorIdxConstant(0, DL));
403 }
404 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
405}
406
407SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
408 EVT EltVT = N->getValueType(0).getVectorElementType();
409 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
410 SDValue LHS = GetScalarizedVector(N->getOperand(0));
411 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
412 LHS, DAG.getValueType(ExtVT));
413}
414
415SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
416 SDLoc DL(N);
417 SDValue Op = N->getOperand(0);
418
419 EVT OpVT = Op.getValueType();
420 EVT OpEltVT = OpVT.getVectorElementType();
421 EVT EltVT = N->getValueType(0).getVectorElementType();
422
423 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
424 Op = GetScalarizedVector(Op);
425 } else {
426 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
427 DAG.getVectorIdxConstant(0, DL));
428 }
429
430 switch (N->getOpcode()) {
432 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
434 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
436 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
437 }
438
439 llvm_unreachable("Illegal extend_vector_inreg opcode");
440}
441
442SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
443 // If the operand is wider than the vector element type then it is implicitly
444 // truncated. Make that explicit here.
445 EVT EltVT = N->getValueType(0).getVectorElementType();
446 SDValue InOp = N->getOperand(0);
447 if (InOp.getValueType() != EltVT)
448 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
449 return InOp;
450}
451
452SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
453 SDValue Cond = N->getOperand(0);
454 EVT OpVT = Cond.getValueType();
455 SDLoc DL(N);
456 // The vselect result and true/value operands needs scalarizing, but it's
457 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
458 // See the similar logic in ScalarizeVecRes_SETCC
459 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
460 Cond = GetScalarizedVector(Cond);
461 } else {
462 EVT VT = OpVT.getVectorElementType();
464 DAG.getVectorIdxConstant(0, DL));
465 }
466
467 SDValue LHS = GetScalarizedVector(N->getOperand(1));
469 TLI.getBooleanContents(false, false);
470 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
471
472 // If integer and float booleans have different contents then we can't
473 // reliably optimize in all cases. There is a full explanation for this in
474 // DAGCombiner::visitSELECT() where the same issue affects folding
475 // (select C, 0, 1) to (xor C, 1).
476 if (TLI.getBooleanContents(false, false) !=
477 TLI.getBooleanContents(false, true)) {
478 // At least try the common case where the boolean is generated by a
479 // comparison.
480 if (Cond->getOpcode() == ISD::SETCC) {
481 EVT OpVT = Cond->getOperand(0).getValueType();
482 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
483 VecBool = TLI.getBooleanContents(OpVT);
484 } else
486 }
487
488 EVT CondVT = Cond.getValueType();
489 if (ScalarBool != VecBool) {
490 switch (ScalarBool) {
492 break;
496 // Vector read from all ones, scalar expects a single 1 so mask.
497 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
498 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
499 break;
503 // Vector reads from a one, scalar from all ones so sign extend.
504 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
505 Cond, DAG.getValueType(MVT::i1));
506 break;
507 }
508 }
509
510 // Truncate the condition if needed
511 auto BoolVT = getSetCCResultType(CondVT);
512 if (BoolVT.bitsLT(CondVT))
513 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
514
515 return DAG.getSelect(SDLoc(N),
516 LHS.getValueType(), Cond, LHS,
517 GetScalarizedVector(N->getOperand(2)));
518}
519
520SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
521 SDValue LHS = GetScalarizedVector(N->getOperand(1));
522 return DAG.getSelect(SDLoc(N),
523 LHS.getValueType(), N->getOperand(0), LHS,
524 GetScalarizedVector(N->getOperand(2)));
525}
526
527SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
528 SDValue LHS = GetScalarizedVector(N->getOperand(2));
529 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
530 N->getOperand(0), N->getOperand(1),
531 LHS, GetScalarizedVector(N->getOperand(3)),
532 N->getOperand(4));
533}
534
535SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
536 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
537}
538
539SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
540 // Figure out if the scalar is the LHS or RHS and return it.
541 SDValue Arg = N->getOperand(2).getOperand(0);
542 if (Arg.isUndef())
543 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
544 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
545 return GetScalarizedVector(N->getOperand(Op));
546}
547
548SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
549 SDValue Src = N->getOperand(0);
550 EVT SrcVT = Src.getValueType();
551 SDLoc dl(N);
552
553 // Handle case where result is scalarized but operand is not
554 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
555 Src = GetScalarizedVector(Src);
556 else
557 Src = DAG.getNode(
559 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
560
561 EVT DstVT = N->getValueType(0).getVectorElementType();
562 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
563}
564
565SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
566 assert(N->getValueType(0).isVector() &&
567 N->getOperand(0).getValueType().isVector() &&
568 "Operand types must be vectors");
569 SDValue LHS = N->getOperand(0);
570 SDValue RHS = N->getOperand(1);
571 EVT OpVT = LHS.getValueType();
572 EVT NVT = N->getValueType(0).getVectorElementType();
573 SDLoc DL(N);
574
575 // The result needs scalarizing, but it's not a given that the source does.
576 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
577 LHS = GetScalarizedVector(LHS);
578 RHS = GetScalarizedVector(RHS);
579 } else {
580 EVT VT = OpVT.getVectorElementType();
581 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
582 DAG.getVectorIdxConstant(0, DL));
583 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
584 DAG.getVectorIdxConstant(0, DL));
585 }
586
587 // Turn it into a scalar SETCC.
588 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
589 N->getOperand(2));
590 // Vectors may have a different boolean contents to scalars. Promote the
591 // value appropriately.
592 ISD::NodeType ExtendCode =
594 return DAG.getNode(ExtendCode, DL, NVT, Res);
595}
596
597SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
598 SDLoc DL(N);
599 SDValue Arg = N->getOperand(0);
600 SDValue Test = N->getOperand(1);
601 EVT ArgVT = Arg.getValueType();
602 EVT ResultVT = N->getValueType(0).getVectorElementType();
603
604 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
605 Arg = GetScalarizedVector(Arg);
606 } else {
607 EVT VT = ArgVT.getVectorElementType();
609 DAG.getVectorIdxConstant(0, DL));
610 }
611
612 SDValue Res =
613 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
614 // Vectors may have a different boolean contents to scalars. Promote the
615 // value appropriately.
616 ISD::NodeType ExtendCode =
618 return DAG.getNode(ExtendCode, DL, ResultVT, Res);
619}
620
621//===----------------------------------------------------------------------===//
622// Operand Vector Scalarization <1 x ty> -> ty.
623//===----------------------------------------------------------------------===//
624
625bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
626 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
627 dbgs() << "\n");
628 SDValue Res = SDValue();
629
630 switch (N->getOpcode()) {
631 default:
632#ifndef NDEBUG
633 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
634 N->dump(&DAG);
635 dbgs() << "\n";
636#endif
637 report_fatal_error("Do not know how to scalarize this operator's "
638 "operand!\n");
639 case ISD::BITCAST:
640 Res = ScalarizeVecOp_BITCAST(N);
641 break;
642 case ISD::ANY_EXTEND:
643 case ISD::ZERO_EXTEND:
644 case ISD::SIGN_EXTEND:
645 case ISD::TRUNCATE:
646 case ISD::FP_TO_SINT:
647 case ISD::FP_TO_UINT:
648 case ISD::SINT_TO_FP:
649 case ISD::UINT_TO_FP:
650 Res = ScalarizeVecOp_UnaryOp(N);
651 break;
656 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
657 break;
659 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
660 break;
662 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
663 break;
664 case ISD::VSELECT:
665 Res = ScalarizeVecOp_VSELECT(N);
666 break;
667 case ISD::SETCC:
668 Res = ScalarizeVecOp_VSETCC(N);
669 break;
670 case ISD::STORE:
671 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
672 break;
674 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
675 break;
676 case ISD::FP_ROUND:
677 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
678 break;
680 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
681 break;
682 case ISD::FP_EXTEND:
683 Res = ScalarizeVecOp_FP_EXTEND(N);
684 break;
698 Res = ScalarizeVecOp_VECREDUCE(N);
699 break;
702 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
703 break;
704 }
705
706 // If the result is null, the sub-method took care of registering results etc.
707 if (!Res.getNode()) return false;
708
709 // If the result is N, the sub-method updated N in place. Tell the legalizer
710 // core about this.
711 if (Res.getNode() == N)
712 return true;
713
714 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
715 "Invalid operand expansion");
716
717 ReplaceValueWith(SDValue(N, 0), Res);
718 return false;
719}
720
721/// If the value to convert is a vector that needs to be scalarized, it must be
722/// <1 x ty>. Convert the element instead.
723SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
724 SDValue Elt = GetScalarizedVector(N->getOperand(0));
725 return DAG.getNode(ISD::BITCAST, SDLoc(N),
726 N->getValueType(0), Elt);
727}
728
729/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
730/// Do the operation on the element instead.
731SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
732 assert(N->getValueType(0).getVectorNumElements() == 1 &&
733 "Unexpected vector type!");
734 SDValue Elt = GetScalarizedVector(N->getOperand(0));
735 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
736 N->getValueType(0).getScalarType(), Elt);
737 // Revectorize the result so the types line up with what the uses of this
738 // expression expect.
739 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
740}
741
742/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
743/// Do the strict FP operation on the element instead.
744SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
745 assert(N->getValueType(0).getVectorNumElements() == 1 &&
746 "Unexpected vector type!");
747 SDValue Elt = GetScalarizedVector(N->getOperand(1));
748 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
749 { N->getValueType(0).getScalarType(), MVT::Other },
750 { N->getOperand(0), Elt });
751 // Legalize the chain result - switch anything that used the old chain to
752 // use the new one.
753 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
754 // Revectorize the result so the types line up with what the uses of this
755 // expression expect.
756 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
757
758 // Do our own replacement and return SDValue() to tell the caller that we
759 // handled all replacements since caller can only handle a single result.
760 ReplaceValueWith(SDValue(N, 0), Res);
761 return SDValue();
762}
763
764/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
765SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
766 SmallVector<SDValue, 8> Ops(N->getNumOperands());
767 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
768 Ops[i] = GetScalarizedVector(N->getOperand(i));
769 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
770}
771
772/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
773/// so just return the element, ignoring the index.
774SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
775 EVT VT = N->getValueType(0);
776 SDValue Res = GetScalarizedVector(N->getOperand(0));
777 if (Res.getValueType() != VT)
778 Res = VT.isFloatingPoint()
779 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
780 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
781 return Res;
782}
783
784/// If the input condition is a vector that needs to be scalarized, it must be
785/// <1 x i1>, so just convert to a normal ISD::SELECT
786/// (still with vector output type since that was acceptable if we got here).
787SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
788 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
789 EVT VT = N->getValueType(0);
790
791 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
792 N->getOperand(2));
793}
794
795/// If the operand is a vector that needs to be scalarized then the
796/// result must be v1i1, so just convert to a scalar SETCC and wrap
797/// with a scalar_to_vector since the res type is legal if we got here
798SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
799 assert(N->getValueType(0).isVector() &&
800 N->getOperand(0).getValueType().isVector() &&
801 "Operand types must be vectors");
802 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
803
804 EVT VT = N->getValueType(0);
805 SDValue LHS = GetScalarizedVector(N->getOperand(0));
806 SDValue RHS = GetScalarizedVector(N->getOperand(1));
807
808 EVT OpVT = N->getOperand(0).getValueType();
809 EVT NVT = VT.getVectorElementType();
810 SDLoc DL(N);
811 // Turn it into a scalar SETCC.
812 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
813 N->getOperand(2));
814
815 // Vectors may have a different boolean contents to scalars. Promote the
816 // value appropriately.
817 ISD::NodeType ExtendCode =
819
820 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
821
822 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
823}
824
825/// If the value to store is a vector that needs to be scalarized, it must be
826/// <1 x ty>. Just store the element.
827SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
828 assert(N->isUnindexed() && "Indexed store of one-element vector?");
829 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
830 SDLoc dl(N);
831
832 if (N->isTruncatingStore())
833 return DAG.getTruncStore(
834 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
835 N->getBasePtr(), N->getPointerInfo(),
836 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
837 N->getMemOperand()->getFlags(), N->getAAInfo());
838
839 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
840 N->getBasePtr(), N->getPointerInfo(),
841 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
842 N->getAAInfo());
843}
844
845/// If the value to round is a vector that needs to be scalarized, it must be
846/// <1 x ty>. Convert the element instead.
847SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
848 assert(OpNo == 0 && "Wrong operand for scalarization!");
849 SDValue Elt = GetScalarizedVector(N->getOperand(0));
850 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
851 N->getValueType(0).getVectorElementType(), Elt,
852 N->getOperand(1));
853 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
854}
855
856SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
857 unsigned OpNo) {
858 assert(OpNo == 1 && "Wrong operand for scalarization!");
859 SDValue Elt = GetScalarizedVector(N->getOperand(1));
861 { N->getValueType(0).getVectorElementType(),
862 MVT::Other },
863 { N->getOperand(0), Elt, N->getOperand(2) });
864 // Legalize the chain result - switch anything that used the old chain to
865 // use the new one.
866 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
867
868 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
869
870 // Do our own replacement and return SDValue() to tell the caller that we
871 // handled all replacements since caller can only handle a single result.
872 ReplaceValueWith(SDValue(N, 0), Res);
873 return SDValue();
874}
875
876/// If the value to extend is a vector that needs to be scalarized, it must be
877/// <1 x ty>. Convert the element instead.
878SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
879 SDValue Elt = GetScalarizedVector(N->getOperand(0));
881 N->getValueType(0).getVectorElementType(), Elt);
882 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
883}
884
885/// If the value to extend is a vector that needs to be scalarized, it must be
886/// <1 x ty>. Convert the element instead.
887SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
888 SDValue Elt = GetScalarizedVector(N->getOperand(1));
889 SDValue Res =
891 {N->getValueType(0).getVectorElementType(), MVT::Other},
892 {N->getOperand(0), Elt});
893 // Legalize the chain result - switch anything that used the old chain to
894 // use the new one.
895 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
896
897 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
898
899 // Do our own replacement and return SDValue() to tell the caller that we
900 // handled all replacements since caller can only handle a single result.
901 ReplaceValueWith(SDValue(N, 0), Res);
902 return SDValue();
903}
904
905SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
906 SDValue Res = GetScalarizedVector(N->getOperand(0));
907 // Result type may be wider than element type.
908 if (Res.getValueType() != N->getValueType(0))
909 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
910 return Res;
911}
912
913SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
914 SDValue AccOp = N->getOperand(0);
915 SDValue VecOp = N->getOperand(1);
916
917 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
918
919 SDValue Op = GetScalarizedVector(VecOp);
920 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
921 AccOp, Op, N->getFlags());
922}
923
924//===----------------------------------------------------------------------===//
925// Result Vector Splitting
926//===----------------------------------------------------------------------===//
927
928/// This method is called when the specified result of the specified node is
929/// found to need vector splitting. At this point, the node may also have
930/// invalid operands or may have other results that need legalization, we just
931/// know that (at least) one result needs vector splitting.
932void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
933 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
934 SDValue Lo, Hi;
935
936 // See if the target wants to custom expand this node.
937 if (CustomLowerNode(N, N->getValueType(ResNo), true))
938 return;
939
940 switch (N->getOpcode()) {
941 default:
942#ifndef NDEBUG
943 dbgs() << "SplitVectorResult #" << ResNo << ": ";
944 N->dump(&DAG);
945 dbgs() << "\n";
946#endif
947 report_fatal_error("Do not know how to split the result of this "
948 "operator!\n");
949
950 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
951 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
952 case ISD::VSELECT:
953 case ISD::SELECT:
954 case ISD::VP_MERGE:
955 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
956 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
957 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
958 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
959 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
960 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
961 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
962 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
963 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
964 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
965 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
966 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
969 SplitVecRes_ScalarOp(N, Lo, Hi);
970 break;
971 case ISD::STEP_VECTOR:
972 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
973 break;
974 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
975 case ISD::LOAD:
976 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
977 break;
978 case ISD::VP_LOAD:
979 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
980 break;
981 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
982 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi);
983 break;
984 case ISD::MLOAD:
985 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
986 break;
987 case ISD::MGATHER:
988 case ISD::VP_GATHER:
989 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
990 break;
991 case ISD::SETCC:
992 case ISD::VP_SETCC:
993 SplitVecRes_SETCC(N, Lo, Hi);
994 break;
996 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
997 break;
999 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
1000 break;
1001 case ISD::VECTOR_SPLICE:
1002 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1003 break;
1005 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1006 return;
1008 SplitVecRes_VECTOR_INTERLEAVE(N);
1009 return;
1010 case ISD::VAARG:
1011 SplitVecRes_VAARG(N, Lo, Hi);
1012 break;
1013
1017 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1018 break;
1019
1020 case ISD::ABS:
1021 case ISD::VP_ABS:
1022 case ISD::BITREVERSE:
1023 case ISD::VP_BITREVERSE:
1024 case ISD::BSWAP:
1025 case ISD::VP_BSWAP:
1026 case ISD::CTLZ:
1027 case ISD::VP_CTLZ:
1028 case ISD::CTTZ:
1029 case ISD::VP_CTTZ:
1031 case ISD::VP_CTLZ_ZERO_UNDEF:
1033 case ISD::VP_CTTZ_ZERO_UNDEF:
1034 case ISD::CTPOP:
1035 case ISD::VP_CTPOP:
1036 case ISD::FABS: case ISD::VP_FABS:
1037 case ISD::FCEIL:
1038 case ISD::VP_FCEIL:
1039 case ISD::FCOS:
1040 case ISD::FEXP:
1041 case ISD::FEXP2:
1042 case ISD::FFLOOR:
1043 case ISD::VP_FFLOOR:
1044 case ISD::FLOG:
1045 case ISD::FLOG10:
1046 case ISD::FLOG2:
1047 case ISD::FNEARBYINT:
1048 case ISD::VP_FNEARBYINT:
1049 case ISD::FNEG: case ISD::VP_FNEG:
1050 case ISD::FREEZE:
1051 case ISD::ARITH_FENCE:
1052 case ISD::FP_EXTEND:
1053 case ISD::VP_FP_EXTEND:
1054 case ISD::FP_ROUND:
1055 case ISD::VP_FP_ROUND:
1056 case ISD::FP_TO_SINT:
1057 case ISD::VP_FP_TO_SINT:
1058 case ISD::FP_TO_UINT:
1059 case ISD::VP_FP_TO_UINT:
1060 case ISD::FRINT:
1061 case ISD::VP_FRINT:
1062 case ISD::FROUND:
1063 case ISD::VP_FROUND:
1064 case ISD::FROUNDEVEN:
1065 case ISD::VP_FROUNDEVEN:
1066 case ISD::FSIN:
1067 case ISD::FSQRT: case ISD::VP_SQRT:
1068 case ISD::FTRUNC:
1069 case ISD::VP_FROUNDTOZERO:
1070 case ISD::SINT_TO_FP:
1071 case ISD::VP_SINT_TO_FP:
1072 case ISD::TRUNCATE:
1073 case ISD::VP_TRUNCATE:
1074 case ISD::UINT_TO_FP:
1075 case ISD::VP_UINT_TO_FP:
1076 case ISD::FCANONICALIZE:
1077 SplitVecRes_UnaryOp(N, Lo, Hi);
1078 break;
1079
1080 case ISD::ANY_EXTEND:
1081 case ISD::SIGN_EXTEND:
1082 case ISD::ZERO_EXTEND:
1083 case ISD::VP_SIGN_EXTEND:
1084 case ISD::VP_ZERO_EXTEND:
1085 SplitVecRes_ExtendOp(N, Lo, Hi);
1086 break;
1087
1088 case ISD::ADD: case ISD::VP_ADD:
1089 case ISD::SUB: case ISD::VP_SUB:
1090 case ISD::MUL: case ISD::VP_MUL:
1091 case ISD::MULHS:
1092 case ISD::MULHU:
1093 case ISD::FADD: case ISD::VP_FADD:
1094 case ISD::FSUB: case ISD::VP_FSUB:
1095 case ISD::FMUL: case ISD::VP_FMUL:
1096 case ISD::FMINNUM: case ISD::VP_FMINNUM:
1097 case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1098 case ISD::FMINIMUM:
1099 case ISD::FMAXIMUM:
1100 case ISD::SDIV: case ISD::VP_SDIV:
1101 case ISD::UDIV: case ISD::VP_UDIV:
1102 case ISD::FDIV: case ISD::VP_FDIV:
1103 case ISD::FPOW:
1104 case ISD::AND: case ISD::VP_AND:
1105 case ISD::OR: case ISD::VP_OR:
1106 case ISD::XOR: case ISD::VP_XOR:
1107 case ISD::SHL: case ISD::VP_SHL:
1108 case ISD::SRA: case ISD::VP_ASHR:
1109 case ISD::SRL: case ISD::VP_LSHR:
1110 case ISD::UREM: case ISD::VP_UREM:
1111 case ISD::SREM: case ISD::VP_SREM:
1112 case ISD::FREM: case ISD::VP_FREM:
1113 case ISD::SMIN: case ISD::VP_SMIN:
1114 case ISD::SMAX: case ISD::VP_SMAX:
1115 case ISD::UMIN: case ISD::VP_UMIN:
1116 case ISD::UMAX: case ISD::VP_UMAX:
1117 case ISD::SADDSAT:
1118 case ISD::UADDSAT:
1119 case ISD::SSUBSAT:
1120 case ISD::USUBSAT:
1121 case ISD::SSHLSAT:
1122 case ISD::USHLSAT:
1123 case ISD::ROTL:
1124 case ISD::ROTR:
1125 case ISD::VP_FCOPYSIGN:
1126 SplitVecRes_BinOp(N, Lo, Hi);
1127 break;
1128 case ISD::FMA: case ISD::VP_FMA:
1129 case ISD::FSHL:
1130 case ISD::VP_FSHL:
1131 case ISD::FSHR:
1132 case ISD::VP_FSHR:
1133 SplitVecRes_TernaryOp(N, Lo, Hi);
1134 break;
1135
1136#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1137 case ISD::STRICT_##DAGN:
1138#include "llvm/IR/ConstrainedOps.def"
1139 SplitVecRes_StrictFPOp(N, Lo, Hi);
1140 break;
1141
1144 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1145 break;
1146
1147 case ISD::UADDO:
1148 case ISD::SADDO:
1149 case ISD::USUBO:
1150 case ISD::SSUBO:
1151 case ISD::UMULO:
1152 case ISD::SMULO:
1153 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1154 break;
1155 case ISD::SMULFIX:
1156 case ISD::SMULFIXSAT:
1157 case ISD::UMULFIX:
1158 case ISD::UMULFIXSAT:
1159 case ISD::SDIVFIX:
1160 case ISD::SDIVFIXSAT:
1161 case ISD::UDIVFIX:
1162 case ISD::UDIVFIXSAT:
1163 SplitVecRes_FIX(N, Lo, Hi);
1164 break;
1165 }
1166
1167 // If Lo/Hi is null, the sub-method took care of registering results etc.
1168 if (Lo.getNode())
1169 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1170}
1171
1172void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1174 uint64_t *ScaledOffset) {
1175 SDLoc DL(N);
1176 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1177
1178 if (MemVT.isScalableVector()) {
1180 SDValue BytesIncrement = DAG.getVScale(
1181 DL, Ptr.getValueType(),
1182 APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1183 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1184 Flags.setNoUnsignedWrap(true);
1185 if (ScaledOffset)
1186 *ScaledOffset += IncrementSize;
1187 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1188 Flags);
1189 } else {
1190 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1191 // Increment the pointer to the other half.
1192 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1193 }
1194}
1195
1196std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1197 return SplitMask(Mask, SDLoc(Mask));
1198}
1199
1200std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1201 const SDLoc &DL) {
1202 SDValue MaskLo, MaskHi;
1203 EVT MaskVT = Mask.getValueType();
1204 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1205 GetSplitVector(Mask, MaskLo, MaskHi);
1206 else
1207 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1208 return std::make_pair(MaskLo, MaskHi);
1209}
1210
1211void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1212 SDValue LHSLo, LHSHi;
1213 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1214 SDValue RHSLo, RHSHi;
1215 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1216 SDLoc dl(N);
1217
1218 const SDNodeFlags Flags = N->getFlags();
1219 unsigned Opcode = N->getOpcode();
1220 if (N->getNumOperands() == 2) {
1221 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1222 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1223 return;
1224 }
1225
1226 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1227 assert(N->isVPOpcode() && "Expected VP opcode");
1228
1229 SDValue MaskLo, MaskHi;
1230 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1231
1232 SDValue EVLLo, EVLHi;
1233 std::tie(EVLLo, EVLHi) =
1234 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1235
1236 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1237 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1238 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1239 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1240}
1241
1242void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1243 SDValue &Hi) {
1244 SDValue Op0Lo, Op0Hi;
1245 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1246 SDValue Op1Lo, Op1Hi;
1247 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1248 SDValue Op2Lo, Op2Hi;
1249 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1250 SDLoc dl(N);
1251
1252 const SDNodeFlags Flags = N->getFlags();
1253 unsigned Opcode = N->getOpcode();
1254 if (N->getNumOperands() == 3) {
1255 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1256 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1257 return;
1258 }
1259
1260 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1261 assert(N->isVPOpcode() && "Expected VP opcode");
1262
1263 SDValue MaskLo, MaskHi;
1264 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1265
1266 SDValue EVLLo, EVLHi;
1267 std::tie(EVLLo, EVLHi) =
1268 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1269
1270 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1271 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1272 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1273 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1274}
1275
1276void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1277 SDValue LHSLo, LHSHi;
1278 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1279 SDValue RHSLo, RHSHi;
1280 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1281 SDLoc dl(N);
1282 SDValue Op2 = N->getOperand(2);
1283
1284 unsigned Opcode = N->getOpcode();
1285 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1286 N->getFlags());
1287 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1288 N->getFlags());
1289}
1290
1291void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1292 SDValue &Hi) {
1293 // We know the result is a vector. The input may be either a vector or a
1294 // scalar value.
1295 EVT LoVT, HiVT;
1296 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1297 SDLoc dl(N);
1298
1299 SDValue InOp = N->getOperand(0);
1300 EVT InVT = InOp.getValueType();
1301
1302 // Handle some special cases efficiently.
1303 switch (getTypeAction(InVT)) {
1311 break;
1314 // A scalar to vector conversion, where the scalar needs expansion.
1315 // If the vector is being split in two then we can just convert the
1316 // expanded pieces.
1317 if (LoVT == HiVT) {
1318 GetExpandedOp(InOp, Lo, Hi);
1319 if (DAG.getDataLayout().isBigEndian())
1320 std::swap(Lo, Hi);
1321 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1322 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1323 return;
1324 }
1325 break;
1327 // If the input is a vector that needs to be split, convert each split
1328 // piece of the input now.
1329 GetSplitVector(InOp, Lo, Hi);
1330 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1331 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1332 return;
1334 report_fatal_error("Scalarization of scalable vectors is not supported.");
1335 }
1336
1337 // In the general case, convert the input to an integer and split it by hand.
1338 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1339 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1340 if (DAG.getDataLayout().isBigEndian())
1341 std::swap(LoIntVT, HiIntVT);
1342
1343 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1344
1345 if (DAG.getDataLayout().isBigEndian())
1346 std::swap(Lo, Hi);
1347 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1348 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1349}
1350
1351void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1352 SDValue &Hi) {
1353 EVT LoVT, HiVT;
1354 SDLoc dl(N);
1355 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1356 unsigned LoNumElts = LoVT.getVectorNumElements();
1357 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1358 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1359
1360 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1361 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1362}
1363
1364void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1365 SDValue &Hi) {
1366 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1367 SDLoc dl(N);
1368 unsigned NumSubvectors = N->getNumOperands() / 2;
1369 if (NumSubvectors == 1) {
1370 Lo = N->getOperand(0);
1371 Hi = N->getOperand(1);
1372 return;
1373 }
1374
1375 EVT LoVT, HiVT;
1376 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1377
1378 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1379 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1380
1381 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1382 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1383}
1384
1385void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1386 SDValue &Hi) {
1387 SDValue Vec = N->getOperand(0);
1388 SDValue Idx = N->getOperand(1);
1389 SDLoc dl(N);
1390
1391 EVT LoVT, HiVT;
1392 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1393
1394 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1395 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1396 Hi = DAG.getNode(
1397 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1398 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1399}
1400
1401void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1402 SDValue &Hi) {
1403 SDValue Vec = N->getOperand(0);
1404 SDValue SubVec = N->getOperand(1);
1405 SDValue Idx = N->getOperand(2);
1406 SDLoc dl(N);
1407 GetSplitVector(Vec, Lo, Hi);
1408
1409 EVT VecVT = Vec.getValueType();
1410 EVT LoVT = Lo.getValueType();
1411 EVT SubVecVT = SubVec.getValueType();
1412 unsigned VecElems = VecVT.getVectorMinNumElements();
1413 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1414 unsigned LoElems = LoVT.getVectorMinNumElements();
1415
1416 // If we know the index is in the first half, and we know the subvector
1417 // doesn't cross the boundary between the halves, we can avoid spilling the
1418 // vector, and insert into the lower half of the split vector directly.
1419 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1420 if (IdxVal + SubElems <= LoElems) {
1421 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1422 return;
1423 }
1424 // Similarly if the subvector is fully in the high half, but mind that we
1425 // can't tell whether a fixed-length subvector is fully within the high half
1426 // of a scalable vector.
1427 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1428 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1429 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1430 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1431 return;
1432 }
1433
1434 // Spill the vector to the stack.
1435 // In cases where the vector is illegal it will be broken down into parts
1436 // and stored in parts - we should use the alignment for the smallest part.
1437 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1439 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1440 auto &MF = DAG.getMachineFunction();
1441 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1442 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1443
1444 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1445 SmallestAlign);
1446
1447 // Store the new subvector into the specified index.
1448 SDValue SubVecPtr =
1449 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1450 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1452
1453 // Load the Lo part from the stack slot.
1454 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1455 SmallestAlign);
1456
1457 // Increment the pointer to the other part.
1458 auto *Load = cast<LoadSDNode>(Lo);
1459 MachinePointerInfo MPI = Load->getPointerInfo();
1460 IncrementPointer(Load, LoVT, MPI, StackPtr);
1461
1462 // Load the Hi part from the stack slot.
1463 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1464}
1465
1466void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1467 SDValue &Hi) {
1468 SDLoc dl(N);
1469 GetSplitVector(N->getOperand(0), Lo, Hi);
1470 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1471 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1472}
1473
1474void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1475 SDValue &Hi) {
1476 SDValue LHSLo, LHSHi;
1477 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1478 SDLoc DL(N);
1479
1480 SDValue RHSLo, RHSHi;
1481 SDValue RHS = N->getOperand(1);
1482 EVT RHSVT = RHS.getValueType();
1483 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1484 GetSplitVector(RHS, RHSLo, RHSHi);
1485 else
1486 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1487
1488
1489 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1490 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1491}
1492
1493void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1494 SDValue &Hi) {
1495 SDLoc DL(N);
1496 SDValue ArgLo, ArgHi;
1497 SDValue Test = N->getOperand(1);
1498 SDValue FpValue = N->getOperand(0);
1499 if (getTypeAction(FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1500 GetSplitVector(FpValue, ArgLo, ArgHi);
1501 else
1502 std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue));
1503 EVT LoVT, HiVT;
1504 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1505
1506 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1507 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1508}
1509
1510void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1511 SDValue &Hi) {
1512 SDValue LHSLo, LHSHi;
1513 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1514 SDLoc dl(N);
1515
1516 EVT LoVT, HiVT;
1517 std::tie(LoVT, HiVT) =
1518 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1519
1520 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1521 DAG.getValueType(LoVT));
1522 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1523 DAG.getValueType(HiVT));
1524}
1525
1526void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1527 SDValue &Hi) {
1528 unsigned Opcode = N->getOpcode();
1529 SDValue N0 = N->getOperand(0);
1530
1531 SDLoc dl(N);
1532 SDValue InLo, InHi;
1533
1534 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1535 GetSplitVector(N0, InLo, InHi);
1536 else
1537 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1538
1539 EVT InLoVT = InLo.getValueType();
1540 unsigned InNumElements = InLoVT.getVectorNumElements();
1541
1542 EVT OutLoVT, OutHiVT;
1543 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1544 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1545 assert((2 * OutNumElements) <= InNumElements &&
1546 "Illegal extend vector in reg split");
1547
1548 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1549 // input vector (i.e. we only use InLo):
1550 // OutLo will extend the first OutNumElements from InLo.
1551 // OutHi will extend the next OutNumElements from InLo.
1552
1553 // Shuffle the elements from InLo for OutHi into the bottom elements to
1554 // create a 'fake' InHi.
1555 SmallVector<int, 8> SplitHi(InNumElements, -1);
1556 for (unsigned i = 0; i != OutNumElements; ++i)
1557 SplitHi[i] = i + OutNumElements;
1558 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1559
1560 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1561 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1562}
1563
1564void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1565 SDValue &Hi) {
1566 unsigned NumOps = N->getNumOperands();
1567 SDValue Chain = N->getOperand(0);
1568 EVT LoVT, HiVT;
1569 SDLoc dl(N);
1570 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1571
1572 SmallVector<SDValue, 4> OpsLo(NumOps);
1573 SmallVector<SDValue, 4> OpsHi(NumOps);
1574
1575 // The Chain is the first operand.
1576 OpsLo[0] = Chain;
1577 OpsHi[0] = Chain;
1578
1579 // Now process the remaining operands.
1580 for (unsigned i = 1; i < NumOps; ++i) {
1581 SDValue Op = N->getOperand(i);
1582 SDValue OpLo = Op;
1583 SDValue OpHi = Op;
1584
1585 EVT InVT = Op.getValueType();
1586 if (InVT.isVector()) {
1587 // If the input also splits, handle it directly for a
1588 // compile time speedup. Otherwise split it by hand.
1589 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1590 GetSplitVector(Op, OpLo, OpHi);
1591 else
1592 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1593 }
1594
1595 OpsLo[i] = OpLo;
1596 OpsHi[i] = OpHi;
1597 }
1598
1599 EVT LoValueVTs[] = {LoVT, MVT::Other};
1600 EVT HiValueVTs[] = {HiVT, MVT::Other};
1601 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1602 N->getFlags());
1603 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1604 N->getFlags());
1605
1606 // Build a factor node to remember that this Op is independent of the
1607 // other one.
1608 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1609 Lo.getValue(1), Hi.getValue(1));
1610
1611 // Legalize the chain result - switch anything that used the old chain to
1612 // use the new one.
1613 ReplaceValueWith(SDValue(N, 1), Chain);
1614}
1615
1616SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1617 SDValue Chain = N->getOperand(0);
1618 EVT VT = N->getValueType(0);
1619 unsigned NE = VT.getVectorNumElements();
1620 EVT EltVT = VT.getVectorElementType();
1621 SDLoc dl(N);
1622
1624 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1625
1626 // If ResNE is 0, fully unroll the vector op.
1627 if (ResNE == 0)
1628 ResNE = NE;
1629 else if (NE > ResNE)
1630 NE = ResNE;
1631
1632 //The results of each unrolled operation, including the chain.
1633 EVT ChainVTs[] = {EltVT, MVT::Other};
1635
1636 unsigned i;
1637 for (i = 0; i != NE; ++i) {
1638 Operands[0] = Chain;
1639 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1640 SDValue Operand = N->getOperand(j);
1641 EVT OperandVT = Operand.getValueType();
1642 if (OperandVT.isVector()) {
1643 EVT OperandEltVT = OperandVT.getVectorElementType();
1644 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1645 Operand, DAG.getVectorIdxConstant(i, dl));
1646 } else {
1647 Operands[j] = Operand;
1648 }
1649 }
1650 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1651 Scalar.getNode()->setFlags(N->getFlags());
1652
1653 //Add in the scalar as well as its chain value to the
1654 //result vectors.
1655 Scalars.push_back(Scalar);
1656 Chains.push_back(Scalar.getValue(1));
1657 }
1658
1659 for (; i < ResNE; ++i)
1660 Scalars.push_back(DAG.getUNDEF(EltVT));
1661
1662 // Build a new factor node to connect the chain back together.
1663 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1664 ReplaceValueWith(SDValue(N, 1), Chain);
1665
1666 // Create a new BUILD_VECTOR node
1667 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1668 return DAG.getBuildVector(VecVT, dl, Scalars);
1669}
1670
1671void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1672 SDValue &Lo, SDValue &Hi) {
1673 SDLoc dl(N);
1674 EVT ResVT = N->getValueType(0);
1675 EVT OvVT = N->getValueType(1);
1676 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1677 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1678 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1679
1680 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1681 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1682 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1683 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1684 } else {
1685 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1686 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1687 }
1688
1689 unsigned Opcode = N->getOpcode();
1690 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1691 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1692 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1693 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1694 LoNode->setFlags(N->getFlags());
1695 HiNode->setFlags(N->getFlags());
1696
1697 Lo = SDValue(LoNode, ResNo);
1698 Hi = SDValue(HiNode, ResNo);
1699
1700 // Replace the other vector result not being explicitly split here.
1701 unsigned OtherNo = 1 - ResNo;
1702 EVT OtherVT = N->getValueType(OtherNo);
1703 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1704 SetSplitVector(SDValue(N, OtherNo),
1705 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1706 } else {
1707 SDValue OtherVal = DAG.getNode(
1708 ISD::CONCAT_VECTORS, dl, OtherVT,
1709 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1710 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1711 }
1712}
1713
1714void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1715 SDValue &Hi) {
1716 SDValue Vec = N->getOperand(0);
1717 SDValue Elt = N->getOperand(1);
1718 SDValue Idx = N->getOperand(2);
1719 SDLoc dl(N);
1720 GetSplitVector(Vec, Lo, Hi);
1721
1722 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1723 unsigned IdxVal = CIdx->getZExtValue();
1724 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1725 if (IdxVal < LoNumElts) {
1727 Lo.getValueType(), Lo, Elt, Idx);
1728 return;
1729 } else if (!Vec.getValueType().isScalableVector()) {
1730 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1731 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1732 return;
1733 }
1734 }
1735
1736 // See if the target wants to custom expand this node.
1737 if (CustomLowerNode(N, N->getValueType(0), true))
1738 return;
1739
1740 // Make the vector elements byte-addressable if they aren't already.
1741 EVT VecVT = Vec.getValueType();
1742 EVT EltVT = VecVT.getVectorElementType();
1743 if (VecVT.getScalarSizeInBits() < 8) {
1744 EltVT = MVT::i8;
1745 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1746 VecVT.getVectorElementCount());
1747 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1748 // Extend the element type to match if needed.
1749 if (EltVT.bitsGT(Elt.getValueType()))
1750 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1751 }
1752
1753 // Spill the vector to the stack.
1754 // In cases where the vector is illegal it will be broken down into parts
1755 // and stored in parts - we should use the alignment for the smallest part.
1756 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1758 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1759 auto &MF = DAG.getMachineFunction();
1760 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1761 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1762
1763 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1764 SmallestAlign);
1765
1766 // Store the new element. This may be larger than the vector element type,
1767 // so use a truncating store.
1768 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1769 Store = DAG.getTruncStore(
1770 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1771 commonAlignment(SmallestAlign,
1772 EltVT.getFixedSizeInBits() / 8));
1773
1774 EVT LoVT, HiVT;
1775 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1776
1777 // Load the Lo part from the stack slot.
1778 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1779
1780 // Increment the pointer to the other part.
1781 auto Load = cast<LoadSDNode>(Lo);
1782 MachinePointerInfo MPI = Load->getPointerInfo();
1783 IncrementPointer(Load, LoVT, MPI, StackPtr);
1784
1785 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1786
1787 // If we adjusted the original type, we need to truncate the results.
1788 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1789 if (LoVT != Lo.getValueType())
1790 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1791 if (HiVT != Hi.getValueType())
1792 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1793}
1794
1795void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1796 SDValue &Hi) {
1797 EVT LoVT, HiVT;
1798 SDLoc dl(N);
1799 assert(N->getValueType(0).isScalableVector() &&
1800 "Only scalable vectors are supported for STEP_VECTOR");
1801 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1802 SDValue Step = N->getOperand(0);
1803
1804 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1805
1806 // Hi = Lo + (EltCnt * Step)
1807 EVT EltVT = Step.getValueType();
1808 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1809 SDValue StartOfHi =
1810 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1811 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1812 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1813
1814 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1815 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1816}
1817
1818void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1819 SDValue &Hi) {
1820 EVT LoVT, HiVT;
1821 SDLoc dl(N);
1822 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1823 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1824 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1825 Hi = DAG.getUNDEF(HiVT);
1826 } else {
1827 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1828 Hi = Lo;
1829 }
1830}
1831
1832void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1833 SDValue &Hi) {
1834 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1835 EVT LoVT, HiVT;
1836 SDLoc dl(LD);
1837 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1838
1839 ISD::LoadExtType ExtType = LD->getExtensionType();
1840 SDValue Ch = LD->getChain();
1841 SDValue Ptr = LD->getBasePtr();
1842 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1843 EVT MemoryVT = LD->getMemoryVT();
1844 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1845 AAMDNodes AAInfo = LD->getAAInfo();
1846
1847 EVT LoMemVT, HiMemVT;
1848 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1849
1850 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1851 SDValue Value, NewChain;
1852 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1853 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1854 ReplaceValueWith(SDValue(LD, 1), NewChain);
1855 return;
1856 }
1857
1858 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1859 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1860 MMOFlags, AAInfo);
1861
1863 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1864
1865 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1866 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1867
1868 // Build a factor node to remember that this load is independent of the
1869 // other one.
1870 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1871 Hi.getValue(1));
1872
1873 // Legalize the chain result - switch anything that used the old chain to
1874 // use the new one.
1875 ReplaceValueWith(SDValue(LD, 1), Ch);
1876}
1877
1878void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1879 SDValue &Hi) {
1880 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1881 EVT LoVT, HiVT;
1882 SDLoc dl(LD);
1883 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1884
1885 ISD::LoadExtType ExtType = LD->getExtensionType();
1886 SDValue Ch = LD->getChain();
1887 SDValue Ptr = LD->getBasePtr();
1888 SDValue Offset = LD->getOffset();
1889 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1890 Align Alignment = LD->getOriginalAlign();
1891 SDValue Mask = LD->getMask();
1892 SDValue EVL = LD->getVectorLength();
1893 EVT MemoryVT = LD->getMemoryVT();
1894
1895 EVT LoMemVT, HiMemVT;
1896 bool HiIsEmpty = false;
1897 std::tie(LoMemVT, HiMemVT) =
1898 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1899
1900 // Split Mask operand
1901 SDValue MaskLo, MaskHi;
1902 if (Mask.getOpcode() == ISD::SETCC) {
1903 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1904 } else {
1905 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1906 GetSplitVector(Mask, MaskLo, MaskHi);
1907 else
1908 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1909 }
1910
1911 // Split EVL operand
1912 SDValue EVLLo, EVLHi;
1913 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
1914
1916 LD->getPointerInfo(), MachineMemOperand::MOLoad,
1917 MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
1918
1919 Lo =
1920 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
1921 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
1922
1923 if (HiIsEmpty) {
1924 // The hi vp_load has zero storage size. We therefore simply set it to
1925 // the low vp_load and rely on subsequent removal from the chain.
1926 Hi = Lo;
1927 } else {
1928 // Generate hi vp_load.
1929 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1930 LD->isExpandingLoad());
1931
1933 if (LoMemVT.isScalableVector())
1934 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1935 else
1936 MPI = LD->getPointerInfo().getWithOffset(
1937 LoMemVT.getStoreSize().getFixedValue());
1938
1941 LD->getAAInfo(), LD->getRanges());
1942
1943 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
1944 Offset, MaskHi, EVLHi, HiMemVT, MMO,
1945 LD->isExpandingLoad());
1946 }
1947
1948 // Build a factor node to remember that this load is independent of the
1949 // other one.
1950 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1951 Hi.getValue(1));
1952
1953 // Legalize the chain result - switch anything that used the old chain to
1954 // use the new one.
1955 ReplaceValueWith(SDValue(LD, 1), Ch);
1956}
1957
1958void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
1959 SDValue &Lo, SDValue &Hi) {
1960 assert(SLD->isUnindexed() &&
1961 "Indexed VP strided load during type legalization!");
1962 assert(SLD->getOffset().isUndef() &&
1963 "Unexpected indexed variable-length load offset");
1964
1965 SDLoc DL(SLD);
1966
1967 EVT LoVT, HiVT;
1968 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0));
1969
1970 EVT LoMemVT, HiMemVT;
1971 bool HiIsEmpty = false;
1972 std::tie(LoMemVT, HiMemVT) =
1973 DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty);
1974
1975 SDValue Mask = SLD->getMask();
1976 SDValue LoMask, HiMask;
1977 if (Mask.getOpcode() == ISD::SETCC) {
1978 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
1979 } else {
1980 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1981 GetSplitVector(Mask, LoMask, HiMask);
1982 else
1983 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1984 }
1985
1986 SDValue LoEVL, HiEVL;
1987 std::tie(LoEVL, HiEVL) =
1988 DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL);
1989
1990 // Generate the low vp_strided_load
1991 Lo = DAG.getStridedLoadVP(
1992 SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL,
1993 SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(),
1994 LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad());
1995
1996 if (HiIsEmpty) {
1997 // The high vp_strided_load has zero storage size. We therefore simply set
1998 // it to the low vp_strided_load and rely on subsequent removal from the
1999 // chain.
2000 Hi = Lo;
2001 } else {
2002 // Generate the high vp_strided_load.
2003 // To calculate the high base address, we need to sum to the low base
2004 // address stride number of bytes for each element already loaded by low,
2005 // that is: Ptr = Ptr + (LoEVL * Stride)
2006 EVT PtrVT = SLD->getBasePtr().getValueType();
2008 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
2009 DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT));
2010 SDValue Ptr =
2011 DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment);
2012
2013 Align Alignment = SLD->getOriginalAlign();
2014 if (LoMemVT.isScalableVector())
2015 Alignment = commonAlignment(
2016 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2017
2021 SLD->getAAInfo(), SLD->getRanges());
2022
2024 HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
2025 SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO,
2026 SLD->isExpandingLoad());
2027 }
2028
2029 // Build a factor node to remember that this load is independent of the
2030 // other one.
2031 SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
2032 Hi.getValue(1));
2033
2034 // Legalize the chain result - switch anything that used the old chain to
2035 // use the new one.
2036 ReplaceValueWith(SDValue(SLD, 1), Ch);
2037}
2038
2039void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2040 SDValue &Lo, SDValue &Hi) {
2041 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2042 EVT LoVT, HiVT;
2043 SDLoc dl(MLD);
2044 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
2045
2046 SDValue Ch = MLD->getChain();
2047 SDValue Ptr = MLD->getBasePtr();
2048 SDValue Offset = MLD->getOffset();
2049 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2050 SDValue Mask = MLD->getMask();
2051 SDValue PassThru = MLD->getPassThru();
2052 Align Alignment = MLD->getOriginalAlign();
2053 ISD::LoadExtType ExtType = MLD->getExtensionType();
2054
2055 // Split Mask operand
2056 SDValue MaskLo, MaskHi;
2057 if (Mask.getOpcode() == ISD::SETCC) {
2058 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2059 } else {
2060 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2061 GetSplitVector(Mask, MaskLo, MaskHi);
2062 else
2063 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2064 }
2065
2066 EVT MemoryVT = MLD->getMemoryVT();
2067 EVT LoMemVT, HiMemVT;
2068 bool HiIsEmpty = false;
2069 std::tie(LoMemVT, HiMemVT) =
2070 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2071
2072 SDValue PassThruLo, PassThruHi;
2073 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2074 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2075 else
2076 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2077
2080 MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
2081 MLD->getRanges());
2082
2083 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
2084 MMO, MLD->getAddressingMode(), ExtType,
2085 MLD->isExpandingLoad());
2086
2087 if (HiIsEmpty) {
2088 // The hi masked load has zero storage size. We therefore simply set it to
2089 // the low masked load and rely on subsequent removal from the chain.
2090 Hi = Lo;
2091 } else {
2092 // Generate hi masked load.
2093 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2094 MLD->isExpandingLoad());
2095
2097 if (LoMemVT.isScalableVector())
2099 else
2100 MPI = MLD->getPointerInfo().getWithOffset(
2101 LoMemVT.getStoreSize().getFixedValue());
2102
2105 MLD->getAAInfo(), MLD->getRanges());
2106
2107 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
2108 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
2109 MLD->isExpandingLoad());
2110 }
2111
2112 // Build a factor node to remember that this load is independent of the
2113 // other one.
2114 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2115 Hi.getValue(1));
2116
2117 // Legalize the chain result - switch anything that used the old chain to
2118 // use the new one.
2119 ReplaceValueWith(SDValue(MLD, 1), Ch);
2120
2121}
2122
2123void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2124 SDValue &Hi, bool SplitSETCC) {
2125 EVT LoVT, HiVT;
2126 SDLoc dl(N);
2127 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2128
2129 SDValue Ch = N->getChain();
2130 SDValue Ptr = N->getBasePtr();
2131 struct Operands {
2132 SDValue Mask;
2133 SDValue Index;
2134 SDValue Scale;
2135 } Ops = [&]() -> Operands {
2136 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2137 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2138 }
2139 auto *VPSC = cast<VPGatherSDNode>(N);
2140 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2141 }();
2142
2143 EVT MemoryVT = N->getMemoryVT();
2144 Align Alignment = N->getOriginalAlign();
2145
2146 // Split Mask operand
2147 SDValue MaskLo, MaskHi;
2148 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2149 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2150 } else {
2151 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2152 }
2153
2154 EVT LoMemVT, HiMemVT;
2155 // Split MemoryVT
2156 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2157
2158 SDValue IndexHi, IndexLo;
2159 if (getTypeAction(Ops.Index.getValueType()) ==
2161 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2162 else
2163 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2164
2166 N->getPointerInfo(), MachineMemOperand::MOLoad,
2167 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2168
2169 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2170 SDValue PassThru = MGT->getPassThru();
2171 SDValue PassThruLo, PassThruHi;
2172 if (getTypeAction(PassThru.getValueType()) ==
2174 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2175 else
2176 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2177
2178 ISD::LoadExtType ExtType = MGT->getExtensionType();
2179 ISD::MemIndexType IndexTy = MGT->getIndexType();
2180
2181 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2182 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2183 OpsLo, MMO, IndexTy, ExtType);
2184
2185 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2186 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2187 OpsHi, MMO, IndexTy, ExtType);
2188 } else {
2189 auto *VPGT = cast<VPGatherSDNode>(N);
2190 SDValue EVLLo, EVLHi;
2191 std::tie(EVLLo, EVLHi) =
2192 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2193
2194 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2195 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2196 MMO, VPGT->getIndexType());
2197
2198 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2199 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2200 MMO, VPGT->getIndexType());
2201 }
2202
2203 // Build a factor node to remember that this load is independent of the
2204 // other one.
2205 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2206 Hi.getValue(1));
2207
2208 // Legalize the chain result - switch anything that used the old chain to
2209 // use the new one.
2210 ReplaceValueWith(SDValue(N, 1), Ch);
2211}
2212
2213void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2214 assert(N->getValueType(0).isVector() &&
2215 N->getOperand(0).getValueType().isVector() &&
2216 "Operand types must be vectors");
2217
2218 EVT LoVT, HiVT;
2219 SDLoc DL(N);
2220 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2221
2222 // If the input also splits, handle it directly. Otherwise split it by hand.
2223 SDValue LL, LH, RL, RH;
2224 if (getTypeAction(N->getOperand(0).getValueType()) ==
2226 GetSplitVector(N->getOperand(0), LL, LH);
2227 else
2228 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2229
2230 if (getTypeAction(N->getOperand(1).getValueType()) ==
2232 GetSplitVector(N->getOperand(1), RL, RH);
2233 else
2234 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2235
2236 if (N->getOpcode() == ISD::SETCC) {
2237 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2238 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2239 } else {
2240 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2241 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2242 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
2243 std::tie(EVLLo, EVLHi) =
2244 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
2245 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo,
2246 EVLLo);
2247 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi,
2248 EVLHi);
2249 }
2250}
2251
2252void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2253 SDValue &Hi) {
2254 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2255 EVT LoVT, HiVT;
2256 SDLoc dl(N);
2257 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2258
2259 // If the input also splits, handle it directly for a compile time speedup.
2260 // Otherwise split it by hand.
2261 EVT InVT = N->getOperand(0).getValueType();
2262 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2263 GetSplitVector(N->getOperand(0), Lo, Hi);
2264 else
2265 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2266
2267 const SDNodeFlags Flags = N->getFlags();
2268 unsigned Opcode = N->getOpcode();
2269 if (N->getNumOperands() <= 2) {
2270 if (Opcode == ISD::FP_ROUND) {
2271 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2272 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2273 } else {
2274 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2275 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2276 }
2277 return;
2278 }
2279
2280 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2281 assert(N->isVPOpcode() && "Expected VP opcode");
2282
2283 SDValue MaskLo, MaskHi;
2284 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2285
2286 SDValue EVLLo, EVLHi;
2287 std::tie(EVLLo, EVLHi) =
2288 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2289
2290 Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2291 Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2292}
2293
2294void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2295 SDValue &Hi) {
2296 SDLoc dl(N);
2297 EVT SrcVT = N->getOperand(0).getValueType();
2298 EVT DestVT = N->getValueType(0);
2299 EVT LoVT, HiVT;
2300 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2301
2302 // We can do better than a generic split operation if the extend is doing
2303 // more than just doubling the width of the elements and the following are
2304 // true:
2305 // - The number of vector elements is even,
2306 // - the source type is legal,
2307 // - the type of a split source is illegal,
2308 // - the type of an extended (by doubling element size) source is legal, and
2309 // - the type of that extended source when split is legal.
2310 //
2311 // This won't necessarily completely legalize the operation, but it will
2312 // more effectively move in the right direction and prevent falling down
2313 // to scalarization in many cases due to the input vector being split too
2314 // far.
2315 if (SrcVT.getVectorElementCount().isKnownEven() &&
2316 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2317 LLVMContext &Ctx = *DAG.getContext();
2318 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2319 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2320
2321 EVT SplitLoVT, SplitHiVT;
2322 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2323 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2324 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2325 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2326 N->dump(&DAG); dbgs() << "\n");
2327 if (!N->isVPOpcode()) {
2328 // Extend the source vector by one step.
2329 SDValue NewSrc =
2330 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2331 // Get the low and high halves of the new, extended one step, vector.
2332 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2333 // Extend those vector halves the rest of the way.
2334 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2335 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2336 return;
2337 }
2338
2339 // Extend the source vector by one step.
2340 SDValue NewSrc =
2341 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0),
2342 N->getOperand(1), N->getOperand(2));
2343 // Get the low and high halves of the new, extended one step, vector.
2344 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2345
2346 SDValue MaskLo, MaskHi;
2347 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2348
2349 SDValue EVLLo, EVLHi;
2350 std::tie(EVLLo, EVLHi) =
2351 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2352 // Extend those vector halves the rest of the way.
2353 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2354 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2355 return;
2356 }
2357 }
2358 // Fall back to the generic unary operator splitting otherwise.
2359 SplitVecRes_UnaryOp(N, Lo, Hi);
2360}
2361
2362void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2363 SDValue &Lo, SDValue &Hi) {
2364 // The low and high parts of the original input give four input vectors.
2365 SDValue Inputs[4];
2366 SDLoc DL(N);
2367 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2368 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2369 EVT NewVT = Inputs[0].getValueType();
2370 unsigned NewElts = NewVT.getVectorNumElements();
2371
2372 auto &&IsConstant = [](const SDValue &N) {
2373 APInt SplatValue;
2374 return N.getResNo() == 0 &&
2375 (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2377 };
2378 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2379 SDValue &Input2,
2381 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2382 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2383 "Expected build vector node.");
2384 EVT EltVT = NewVT.getVectorElementType();
2385 SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2386 for (unsigned I = 0; I < NewElts; ++I) {
2387 if (Mask[I] == PoisonMaskElem)
2388 continue;
2389 unsigned Idx = Mask[I];
2390 if (Idx >= NewElts)
2391 Ops[I] = Input2.getOperand(Idx - NewElts);
2392 else
2393 Ops[I] = Input1.getOperand(Idx);
2394 // Make the type of all elements the same as the element type.
2395 if (Ops[I].getValueType().bitsGT(EltVT))
2396 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2397 }
2398 return DAG.getBuildVector(NewVT, DL, Ops);
2399 };
2400
2401 // If Lo or Hi uses elements from at most two of the four input vectors, then
2402 // express it as a vector shuffle of those two inputs. Otherwise extract the
2403 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2404 SmallVector<int> OrigMask(N->getMask());
2405 // Try to pack incoming shuffles/inputs.
2406 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2408 // Check if all inputs are shuffles of the same operands or non-shuffles.
2410 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2411 SDValue Input = Inputs[Idx];
2412 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2413 if (!Shuffle ||
2414 Input.getOperand(0).getValueType() != Input.getValueType())
2415 continue;
2416 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2417 .push_back(Idx);
2418 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2419 .push_back(Idx);
2420 }
2421 for (auto &P : ShufflesIdxs) {
2422 if (P.second.size() < 2)
2423 continue;
2424 // Use shuffles operands instead of shuffles themselves.
2425 // 1. Adjust mask.
2426 for (int &Idx : Mask) {
2427 if (Idx == PoisonMaskElem)
2428 continue;
2429 unsigned SrcRegIdx = Idx / NewElts;
2430 if (Inputs[SrcRegIdx].isUndef()) {
2432 continue;
2433 }
2434 auto *Shuffle =
2435 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2436 if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2437 continue;
2438 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2439 if (MaskElt == PoisonMaskElem) {
2441 continue;
2442 }
2443 Idx = MaskElt % NewElts +
2444 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2445 ? 0
2446 : 1] *
2447 NewElts;
2448 }
2449 // 2. Update inputs.
2450 Inputs[P.second[0]] = P.first.first;
2451 Inputs[P.second[1]] = P.first.second;
2452 // Clear the pair data.
2453 P.second.clear();
2454 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2455 }
2456 // Check if any concat_vectors can be simplified.
2457 SmallBitVector UsedSubVector(2 * std::size(Inputs));
2458 for (int &Idx : Mask) {
2459 if (Idx == PoisonMaskElem)
2460 continue;
2461 unsigned SrcRegIdx = Idx / NewElts;
2462 if (Inputs[SrcRegIdx].isUndef()) {
2464 continue;
2465 }
2467 getTypeAction(Inputs[SrcRegIdx].getValueType());
2468 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2469 Inputs[SrcRegIdx].getNumOperands() == 2 &&
2470 !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2471 (TypeAction == TargetLowering::TypeLegal ||
2472 TypeAction == TargetLowering::TypeWidenVector))
2473 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2474 }
2475 if (UsedSubVector.count() > 1) {
2477 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2478 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2479 continue;
2480 if (Pairs.empty() || Pairs.back().size() == 2)
2481 Pairs.emplace_back();
2482 if (UsedSubVector.test(2 * I)) {
2483 Pairs.back().emplace_back(I, 0);
2484 } else {
2485 assert(UsedSubVector.test(2 * I + 1) &&
2486 "Expected to be used one of the subvectors.");
2487 Pairs.back().emplace_back(I, 1);
2488 }
2489 }
2490 if (!Pairs.empty() && Pairs.front().size() > 1) {
2491 // Adjust mask.
2492 for (int &Idx : Mask) {
2493 if (Idx == PoisonMaskElem)
2494 continue;
2495 unsigned SrcRegIdx = Idx / NewElts;
2496 auto *It = find_if(
2497 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2498 return Idxs.front().first == SrcRegIdx ||
2499 Idxs.back().first == SrcRegIdx;
2500 });
2501 if (It == Pairs.end())
2502 continue;
2503 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2504 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2505 }
2506 // Adjust inputs.
2507 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2508 Inputs[Idxs.front().first] = DAG.getNode(
2510 Inputs[Idxs.front().first].getValueType(),
2511 Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2512 Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2513 }
2514 }
2515 }
2516 bool Changed;
2517 do {
2518 // Try to remove extra shuffles (except broadcasts) and shuffles with the
2519 // reused operands.
2520 Changed = false;
2521 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2522 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2523 if (!Shuffle)
2524 continue;
2525 if (Shuffle->getOperand(0).getValueType() != NewVT)
2526 continue;
2527 int Op = -1;
2528 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2529 !Shuffle->isSplat()) {
2530 Op = 0;
2531 } else if (!Inputs[I].hasOneUse() &&
2532 !Shuffle->getOperand(1).isUndef()) {
2533 // Find the only used operand, if possible.
2534 for (int &Idx : Mask) {
2535 if (Idx == PoisonMaskElem)
2536 continue;
2537 unsigned SrcRegIdx = Idx / NewElts;
2538 if (SrcRegIdx != I)
2539 continue;
2540 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2541 if (MaskElt == PoisonMaskElem) {
2543 continue;
2544 }
2545 int OpIdx = MaskElt / NewElts;
2546 if (Op == -1) {
2547 Op = OpIdx;
2548 continue;
2549 }
2550 if (Op != OpIdx) {
2551 Op = -1;
2552 break;
2553 }
2554 }
2555 }
2556 if (Op < 0) {
2557 // Try to check if one of the shuffle operands is used already.
2558 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2559 if (Shuffle->getOperand(OpIdx).isUndef())
2560 continue;
2561 auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2562 if (It == std::end(Inputs))
2563 continue;
2564 int FoundOp = std::distance(std::begin(Inputs), It);
2565 // Found that operand is used already.
2566 // 1. Fix the mask for the reused operand.
2567 for (int &Idx : Mask) {
2568 if (Idx == PoisonMaskElem)
2569 continue;
2570 unsigned SrcRegIdx = Idx / NewElts;
2571 if (SrcRegIdx != I)
2572 continue;
2573 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2574 if (MaskElt == PoisonMaskElem) {
2576 continue;
2577 }
2578 int MaskIdx = MaskElt / NewElts;
2579 if (OpIdx == MaskIdx)
2580 Idx = MaskElt % NewElts + FoundOp * NewElts;
2581 }
2582 // 2. Set Op to the unused OpIdx.
2583 Op = (OpIdx + 1) % 2;
2584 break;
2585 }
2586 }
2587 if (Op >= 0) {
2588 Changed = true;
2589 Inputs[I] = Shuffle->getOperand(Op);
2590 // Adjust mask.
2591 for (int &Idx : Mask) {
2592 if (Idx == PoisonMaskElem)
2593 continue;
2594 unsigned SrcRegIdx = Idx / NewElts;
2595 if (SrcRegIdx != I)
2596 continue;
2597 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2598 int OpIdx = MaskElt / NewElts;
2599 if (OpIdx != Op)
2600 continue;
2601 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2602 }
2603 }
2604 }
2605 } while (Changed);
2606 };
2607 TryPeekThroughShufflesInputs(OrigMask);
2608 // Proces unique inputs.
2609 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2610 NewElts](SmallVectorImpl<int> &Mask) {
2611 SetVector<SDValue> UniqueInputs;
2612 SetVector<SDValue> UniqueConstantInputs;
2613 for (const auto &I : Inputs) {
2614 if (IsConstant(I))
2615 UniqueConstantInputs.insert(I);
2616 else if (!I.isUndef())
2617 UniqueInputs.insert(I);
2618 }
2619 // Adjust mask in case of reused inputs. Also, need to insert constant
2620 // inputs at first, otherwise it affects the final outcome.
2621 if (UniqueInputs.size() != std::size(Inputs)) {
2622 auto &&UniqueVec = UniqueInputs.takeVector();
2623 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2624 unsigned ConstNum = UniqueConstantVec.size();
2625 for (int &Idx : Mask) {
2626 if (Idx == PoisonMaskElem)
2627 continue;
2628 unsigned SrcRegIdx = Idx / NewElts;
2629 if (Inputs[SrcRegIdx].isUndef()) {
2631 continue;
2632 }
2633 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2634 if (It != UniqueConstantVec.end()) {
2635 Idx = (Idx % NewElts) +
2636 NewElts * std::distance(UniqueConstantVec.begin(), It);
2637 assert(Idx >= 0 && "Expected defined mask idx.");
2638 continue;
2639 }
2640 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2641 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2642 Idx = (Idx % NewElts) +
2643 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2644 assert(Idx >= 0 && "Expected defined mask idx.");
2645 }
2646 copy(UniqueConstantVec, std::begin(Inputs));
2647 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2648 }
2649 };
2650 MakeUniqueInputs(OrigMask);
2651 SDValue OrigInputs[4];
2652 copy(Inputs, std::begin(OrigInputs));
2653 for (unsigned High = 0; High < 2; ++High) {
2654 SDValue &Output = High ? Hi : Lo;
2655
2656 // Build a shuffle mask for the output, discovering on the fly which
2657 // input vectors to use as shuffle operands.
2658 unsigned FirstMaskIdx = High * NewElts;
2659 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
2660 copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2661 assert(!Output && "Expected default initialized initial value.");
2662 TryPeekThroughShufflesInputs(Mask);
2663 MakeUniqueInputs(Mask);
2664 SDValue TmpInputs[4];
2665 copy(Inputs, std::begin(TmpInputs));
2666 // Track changes in the output registers.
2667 int UsedIdx = -1;
2668 bool SecondIteration = false;
2669 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2670 if (UsedIdx < 0) {
2671 UsedIdx = Idx;
2672 return false;
2673 }
2674 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2675 SecondIteration = true;
2676 return SecondIteration;
2677 };
2679 Mask, std::size(Inputs), std::size(Inputs),
2680 /*NumOfUsedRegs=*/1,
2681 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2682 [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2683 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2684 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2685 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2686 else
2687 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2688 DAG.getUNDEF(NewVT), Mask);
2689 Inputs[Idx] = Output;
2690 },
2691 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2692 &TmpInputs,
2693 &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2694 if (AccumulateResults(Idx1)) {
2695 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2696 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2697 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2698 else
2699 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2700 Inputs[Idx2], Mask);
2701 } else {
2702 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2703 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2704 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2705 else
2706 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2707 TmpInputs[Idx2], Mask);
2708 }
2709 Inputs[Idx1] = Output;
2710 });
2711 copy(OrigInputs, std::begin(Inputs));
2712 }
2713}
2714
2715void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2716 EVT OVT = N->getValueType(0);
2717 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2718 SDValue Chain = N->getOperand(0);
2719 SDValue Ptr = N->getOperand(1);
2720 SDValue SV = N->getOperand(2);
2721 SDLoc dl(N);
2722
2723 const Align Alignment =
2724 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2725
2726 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2727 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2728 Chain = Hi.getValue(1);
2729
2730 // Modified the chain - switch anything that used the old chain to use
2731 // the new one.
2732 ReplaceValueWith(SDValue(N, 1), Chain);
2733}
2734
2735void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2736 SDValue &Hi) {
2737 EVT DstVTLo, DstVTHi;
2738 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2739 SDLoc dl(N);
2740
2741 SDValue SrcLo, SrcHi;
2742 EVT SrcVT = N->getOperand(0).getValueType();
2743 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2744 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2745 else
2746 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2747
2748 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2749 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2750}
2751
2752void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2753 SDValue &Hi) {
2754 SDValue InLo, InHi;
2755 GetSplitVector(N->getOperand(0), InLo, InHi);
2756 SDLoc DL(N);
2757
2758 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2759 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2760}
2761
2762void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2763 SDValue &Hi) {
2764 EVT VT = N->getValueType(0);
2765 SDLoc DL(N);
2766
2767 EVT LoVT, HiVT;
2768 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2769
2770 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2771 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
2772 DAG.getVectorIdxConstant(0, DL));
2773 Hi =
2774 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
2775 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2776}
2777
2778void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
2779
2780 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2781 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
2782 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
2783 EVT VT = Op0Lo.getValueType();
2784 SDLoc DL(N);
2786 DAG.getVTList(VT, VT), Op0Lo, Op0Hi);
2788 DAG.getVTList(VT, VT), Op1Lo, Op1Hi);
2789
2790 SetSplitVector(SDValue(N, 0), ResLo.getValue(0), ResHi.getValue(0));
2791 SetSplitVector(SDValue(N, 1), ResLo.getValue(1), ResHi.getValue(1));
2792}
2793
2794void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
2795 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2796 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
2797 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
2798 EVT VT = Op0Lo.getValueType();
2799 SDLoc DL(N);
2801 DAG.getVTList(VT, VT), Op0Lo, Op1Lo),
2802 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
2803 DAG.getVTList(VT, VT), Op0Hi, Op1Hi)};
2804
2805 SetSplitVector(SDValue(N, 0), Res[0].getValue(0), Res[0].getValue(1));
2806 SetSplitVector(SDValue(N, 1), Res[1].getValue(0), Res[1].getValue(1));
2807}
2808
2809//===----------------------------------------------------------------------===//
2810// Operand Vector Splitting
2811//===----------------------------------------------------------------------===//
2812
2813/// This method is called when the specified operand of the specified node is
2814/// found to need vector splitting. At this point, all of the result types of
2815/// the node are known to be legal, but other operands of the node may need
2816/// legalization as well as the specified one.
2817bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2818 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2819 SDValue Res = SDValue();
2820
2821 // See if the target wants to custom split this node.
2822 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2823 return false;
2824
2825 switch (N->getOpcode()) {
2826 default:
2827#ifndef NDEBUG
2828 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2829 N->dump(&DAG);
2830 dbgs() << "\n";
2831#endif
2832 report_fatal_error("Do not know how to split this operator's "
2833 "operand!\n");
2834
2835 case ISD::VP_SETCC:
2836 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2837 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2838 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2839 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2840 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2841 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2842 case ISD::VP_TRUNCATE:
2843 case ISD::TRUNCATE:
2844 Res = SplitVecOp_TruncateHelper(N);
2845 break;
2847 case ISD::VP_FP_ROUND:
2848 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2849 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2850 case ISD::STORE:
2851 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2852 break;
2853 case ISD::VP_STORE:
2854 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2855 break;
2856 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2857 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo);
2858 break;
2859 case ISD::MSTORE:
2860 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2861 break;
2862 case ISD::MSCATTER:
2863 case ISD::VP_SCATTER:
2864 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
2865 break;
2866 case ISD::MGATHER:
2867 case ISD::VP_GATHER:
2868 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
2869 break;
2870 case ISD::VSELECT:
2871 Res = SplitVecOp_VSELECT(N, OpNo);
2872 break;
2875 case ISD::SINT_TO_FP:
2876 case ISD::UINT_TO_FP:
2877 case ISD::VP_SINT_TO_FP:
2878 case ISD::VP_UINT_TO_FP:
2879 if (N->getValueType(0).bitsLT(
2880 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2881 Res = SplitVecOp_TruncateHelper(N);
2882 else
2883 Res = SplitVecOp_UnaryOp(N);
2884 break;
2887 Res = SplitVecOp_FP_TO_XINT_SAT(N);
2888 break;
2889 case ISD::FP_TO_SINT:
2890 case ISD::FP_TO_UINT:
2891 case ISD::VP_FP_TO_SINT:
2892 case ISD::VP_FP_TO_UINT:
2896 case ISD::FP_EXTEND:
2897 case ISD::SIGN_EXTEND:
2898 case ISD::ZERO_EXTEND:
2899 case ISD::ANY_EXTEND:
2900 case ISD::FTRUNC:
2901 Res = SplitVecOp_UnaryOp(N);
2902 break;
2903
2907 Res = SplitVecOp_ExtVecInRegOp(N);
2908 break;
2909
2912 case ISD::VECREDUCE_ADD:
2913 case ISD::VECREDUCE_MUL:
2914 case ISD::VECREDUCE_AND:
2915 case ISD::VECREDUCE_OR:
2916 case ISD::VECREDUCE_XOR:
2923 Res = SplitVecOp_VECREDUCE(N, OpNo);
2924 break;
2927 Res = SplitVecOp_VECREDUCE_SEQ(N);
2928 break;
2929 case ISD::VP_REDUCE_FADD:
2930 case ISD::VP_REDUCE_SEQ_FADD:
2931 case ISD::VP_REDUCE_FMUL:
2932 case ISD::VP_REDUCE_SEQ_FMUL:
2933 case ISD::VP_REDUCE_ADD:
2934 case ISD::VP_REDUCE_MUL:
2935 case ISD::VP_REDUCE_AND:
2936 case ISD::VP_REDUCE_OR:
2937 case ISD::VP_REDUCE_XOR:
2938 case ISD::VP_REDUCE_SMAX:
2939 case ISD::VP_REDUCE_SMIN:
2940 case ISD::VP_REDUCE_UMAX:
2941 case ISD::VP_REDUCE_UMIN:
2942 case ISD::VP_REDUCE_FMAX:
2943 case ISD::VP_REDUCE_FMIN:
2944 Res = SplitVecOp_VP_REDUCE(N, OpNo);
2945 break;
2946 }
2947
2948 // If the result is null, the sub-method took care of registering results etc.
2949 if (!Res.getNode()) return false;
2950
2951 // If the result is N, the sub-method updated N in place. Tell the legalizer
2952 // core about this.
2953 if (Res.getNode() == N)
2954 return true;
2955
2956 if (N->isStrictFPOpcode())
2957 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2958 "Invalid operand expansion");
2959 else
2960 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2961 "Invalid operand expansion");
2962
2963 ReplaceValueWith(SDValue(N, 0), Res);
2964 return false;
2965}
2966
2967SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2968 // The only possibility for an illegal operand is the mask, since result type
2969 // legalization would have handled this node already otherwise.
2970 assert(OpNo == 0 && "Illegal operand must be mask");
2971
2972 SDValue Mask = N->getOperand(0);
2973 SDValue Src0 = N->getOperand(1);
2974 SDValue Src1 = N->getOperand(2);
2975 EVT Src0VT = Src0.getValueType();
2976 SDLoc DL(N);
2977 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2978
2979 SDValue Lo, Hi;
2980 GetSplitVector(N->getOperand(0), Lo, Hi);
2981 assert(Lo.getValueType() == Hi.getValueType() &&
2982 "Lo and Hi have differing types");
2983
2984 EVT LoOpVT, HiOpVT;
2985 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2986 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2987
2988 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2989 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2990 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2991 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2992
2993 SDValue LoSelect =
2994 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2995 SDValue HiSelect =
2996 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2997
2998 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2999}
3000
3001SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3002 EVT ResVT = N->getValueType(0);
3003 SDValue Lo, Hi;
3004 SDLoc dl(N);
3005
3006 SDValue VecOp = N->getOperand(OpNo);
3007 EVT VecVT = VecOp.getValueType();
3008 assert(VecVT.isVector() && "Can only split reduce vector operand");
3009 GetSplitVector(VecOp, Lo, Hi);
3010 EVT LoOpVT, HiOpVT;
3011 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3012
3013 // Use the appropriate scalar instruction on the split subvectors before
3014 // reducing the now partially reduced smaller vector.
3015 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
3016 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
3017 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
3018}
3019
3020SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3021 EVT ResVT = N->getValueType(0);
3022 SDValue Lo, Hi;
3023 SDLoc dl(N);
3024
3025 SDValue AccOp = N->getOperand(0);
3026 SDValue VecOp = N->getOperand(1);
3027 SDNodeFlags Flags = N->getFlags();
3028
3029 EVT VecVT = VecOp.getValueType();
3030 assert(VecVT.isVector() && "Can only split reduce vector operand");
3031 GetSplitVector(VecOp, Lo, Hi);
3032 EVT LoOpVT, HiOpVT;
3033 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3034
3035 // Reduce low half.
3036 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
3037
3038 // Reduce high half, using low half result as initial value.
3039 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
3040}
3041
3042SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3043 assert(N->isVPOpcode() && "Expected VP opcode");
3044 assert(OpNo == 1 && "Can only split reduce vector operand");
3045
3046 unsigned Opc = N->getOpcode();
3047 EVT ResVT = N->getValueType(0);
3048 SDValue Lo, Hi;
3049 SDLoc dl(N);
3050
3051 SDValue VecOp = N->getOperand(OpNo);
3052 EVT VecVT = VecOp.getValueType();
3053 assert(VecVT.isVector() && "Can only split reduce vector operand");
3054 GetSplitVector(VecOp, Lo, Hi);
3055
3056 SDValue MaskLo, MaskHi;
3057 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
3058
3059 SDValue EVLLo, EVLHi;
3060 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
3061
3062 const SDNodeFlags Flags = N->getFlags();
3063
3064 SDValue ResLo =
3065 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
3066 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
3067}
3068
3069SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3070 // The result has a legal vector type, but the input needs splitting.
3071 EVT ResVT = N->getValueType(0);
3072 SDValue Lo, Hi;
3073 SDLoc dl(N);
3074 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3075 EVT InVT = Lo.getValueType();
3076
3077 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3078 InVT.getVectorElementCount());
3079
3080 if (N->isStrictFPOpcode()) {
3081 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3082 { N->getOperand(0), Lo });
3083 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3084 { N->getOperand(0), Hi });
3085
3086 // Build a factor node to remember that this operation is independent
3087 // of the other one.
3088 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3089 Hi.getValue(1));
3090
3091 // Legalize the chain result - switch anything that used the old chain to
3092 // use the new one.
3093 ReplaceValueWith(SDValue(N, 1), Ch);
3094 } else if (N->getNumOperands() == 3) {
3095 assert(N->isVPOpcode() && "Expected VP opcode");
3096 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3097 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3098 std::tie(EVLLo, EVLHi) =
3099 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
3100 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo);
3101 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi);
3102 } else {
3103 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
3104 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
3105 }
3106
3107 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3108}
3109
3110SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3111 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3112 // end up being split all the way down to individual components. Convert the
3113 // split pieces into integers and reassemble.
3114 SDValue Lo, Hi;
3115 GetSplitVector(N->getOperand(0), Lo, Hi);
3116 Lo = BitConvertToInteger(Lo);
3117 Hi = BitConvertToInteger(Hi);
3118
3119 if (DAG.getDataLayout().isBigEndian())
3120 std::swap(Lo, Hi);
3121
3122 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
3123 JoinIntegers(Lo, Hi));
3124}
3125
3126SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3127 unsigned OpNo) {
3128 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3129 // We know that the result type is legal.
3130 EVT ResVT = N->getValueType(0);
3131
3132 SDValue Vec = N->getOperand(0);
3133 SDValue SubVec = N->getOperand(1);
3134 SDValue Idx = N->getOperand(2);
3135 SDLoc dl(N);
3136
3137 SDValue Lo, Hi;
3138 GetSplitVector(SubVec, Lo, Hi);
3139
3140 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3141 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3142
3143 SDValue FirstInsertion =
3144 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
3145 SDValue SecondInsertion =
3146 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
3147 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3148
3149 return SecondInsertion;
3150}
3151
3152SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3153 // We know that the extracted result type is legal.
3154 EVT SubVT = N->getValueType(0);
3155 SDValue Idx = N->getOperand(1);
3156 SDLoc dl(N);
3157 SDValue Lo, Hi;
3158
3159 GetSplitVector(N->getOperand(0), Lo, Hi);
3160
3161 uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3162 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3163
3164 if (IdxVal < LoEltsMin) {
3165 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3166 "Extracted subvector crosses vector split!");
3167 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
3168 } else if (SubVT.isScalableVector() ==
3169 N->getOperand(0).getValueType().isScalableVector())
3170 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
3171 DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3172
3173 // After this point the DAG node only permits extracting fixed-width
3174 // subvectors from scalable vectors.
3175 assert(SubVT.isFixedLengthVector() &&
3176 "Extracting scalable subvector from fixed-width unsupported");
3177
3178 // If the element type is i1 and we're not promoting the result, then we may
3179 // end up loading the wrong data since the bits are packed tightly into
3180 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3181 // type at index 4, then we will load a byte starting at index 0.
3182 if (SubVT.getScalarType() == MVT::i1)
3183 report_fatal_error("Don't know how to extract fixed-width predicate "
3184 "subvector from a scalable predicate vector");
3185
3186 // Spill the vector to the stack. We should use the alignment for
3187 // the smallest part.
3188 SDValue Vec = N->getOperand(0);
3189 EVT VecVT = Vec.getValueType();
3190 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3192 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3193 auto &MF = DAG.getMachineFunction();
3194 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3195 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3196
3197 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3198 SmallestAlign);
3199
3200 // Extract the subvector by loading the correct part.
3201 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
3202
3203 return DAG.getLoad(
3204 SubVT, dl, Store, StackPtr,
3205 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3206}
3207
3208SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3209 SDValue Vec = N->getOperand(0);
3210 SDValue Idx = N->getOperand(1);
3211 EVT VecVT = Vec.getValueType();
3212
3213 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) {
3214 uint64_t IdxVal = Index->getZExtValue();
3215
3216 SDValue Lo, Hi;
3217 GetSplitVector(Vec, Lo, Hi);
3218
3219 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3220
3221 if (IdxVal < LoElts)
3222 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
3223 else if (!Vec.getValueType().isScalableVector())
3224 return SDValue(DAG.UpdateNodeOperands(N, Hi,
3225 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
3226 Idx.getValueType())), 0);
3227 }
3228
3229 // See if the target wants to custom expand this node.
3230 if (CustomLowerNode(N, N->getValueType(0), true))
3231 return SDValue();
3232
3233 // Make the vector elements byte-addressable if they aren't already.
3234 SDLoc dl(N);
3235 EVT EltVT = VecVT.getVectorElementType();
3236 if (VecVT.getScalarSizeInBits() < 8) {
3237 EltVT = MVT::i8;
3238 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3239 VecVT.getVectorElementCount());
3240 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3241 }
3242
3243 // Store the vector to the stack.
3244 // In cases where the vector is illegal it will be broken down into parts
3245 // and stored in parts - we should use the alignment for the smallest part.
3246 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3248 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3249 auto &MF = DAG.getMachineFunction();
3250 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3251 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3252 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3253 SmallestAlign);
3254
3255 // Load back the required element.
3256 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3257
3258 // FIXME: This is to handle i1 vectors with elements promoted to i8.
3259 // i1 vector handling needs general improvement.
3260 if (N->getValueType(0).bitsLT(EltVT)) {
3261 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3262 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3263 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3264 }
3265
3266 return DAG.getExtLoad(
3267 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3268 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3269 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3270}
3271
3272SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3273 SDValue Lo, Hi;
3274
3275 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3276 // splitting the result has the same effect as splitting the input operand.
3277 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3278
3279 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3280}
3281
3282SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3283 (void)OpNo;
3284 SDValue Lo, Hi;
3285 SplitVecRes_Gather(N, Lo, Hi);
3286
3287 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3288 ReplaceValueWith(SDValue(N, 0), Res);
3289 return SDValue();
3290}
3291
3292SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3293 assert(N->isUnindexed() && "Indexed vp_store of vector?");
3294 SDValue Ch = N->getChain();
3295 SDValue Ptr = N->getBasePtr();
3296 SDValue Offset = N->getOffset();
3297 assert(Offset.isUndef() && "Unexpected VP store offset");
3298 SDValue Mask = N->getMask();
3299 SDValue EVL = N->getVectorLength();
3300 SDValue Data = N->getValue();
3301 Align Alignment = N->getOriginalAlign();
3302 SDLoc DL(N);
3303
3304 SDValue DataLo, DataHi;
3305 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3306 // Split Data operand
3307 GetSplitVector(Data, DataLo, DataHi);
3308 else
3309 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3310
3311 // Split Mask operand
3312 SDValue MaskLo, MaskHi;
3313 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3314 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3315 } else {
3316 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3317 GetSplitVector(Mask, MaskLo, MaskHi);
3318 else
3319 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3320 }
3321
3322 EVT MemoryVT = N->getMemoryVT();
3323 EVT LoMemVT, HiMemVT;
3324 bool HiIsEmpty = false;
3325 std::tie(LoMemVT, HiMemVT) =
3326 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3327
3328 // Split EVL
3329 SDValue EVLLo, EVLHi;
3330 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3331
3332 SDValue Lo, Hi;
3333 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3334 N->getPointerInfo(), MachineMemOperand::MOStore,
3335 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3336
3337 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3338 N->getAddressingMode(), N->isTruncatingStore(),
3339 N->isCompressingStore());
3340
3341 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3342 if (HiIsEmpty)
3343 return Lo;
3344
3345 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3346 N->isCompressingStore());
3347
3349 if (LoMemVT.isScalableVector()) {
3350 Alignment = commonAlignment(Alignment,
3351 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3352 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3353 } else
3354 MPI = N->getPointerInfo().getWithOffset(
3355 LoMemVT.getStoreSize().getFixedValue());
3356
3357 MMO = DAG.getMachineFunction().getMachineMemOperand(
3359 N->getAAInfo(), N->getRanges());
3360
3361 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3362 N->getAddressingMode(), N->isTruncatingStore(),
3363 N->isCompressingStore());
3364
3365 // Build a factor node to remember that this store is independent of the
3366 // other one.
3367 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3368}
3369
3370SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3371 unsigned OpNo) {
3372 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3373 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3374
3375 SDLoc DL(N);
3376
3377 SDValue Data = N->getValue();
3378 SDValue LoData, HiData;
3379 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3380 GetSplitVector(Data, LoData, HiData);
3381 else
3382 std::tie(LoData, HiData) = DAG.SplitVector(Data, DL);
3383
3384 EVT LoMemVT, HiMemVT;
3385 bool HiIsEmpty = false;
3386 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3387 N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty);
3388
3389 SDValue Mask = N->getMask();
3390 SDValue LoMask, HiMask;
3391 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3392 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
3393 else if (getTypeAction(Mask.getValueType()) ==
3395 GetSplitVector(Mask, LoMask, HiMask);
3396 else
3397 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3398
3399 SDValue LoEVL, HiEVL;
3400 std::tie(LoEVL, HiEVL) =
3401 DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL);
3402
3403 // Generate the low vp_strided_store
3404 SDValue Lo = DAG.getStridedStoreVP(
3405 N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(),
3406 N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(),
3407 N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore());
3408
3409 // If the high vp_strided_store has zero storage size, only the low
3410 // vp_strided_store is needed.
3411 if (HiIsEmpty)
3412 return Lo;
3413
3414 // Generate the high vp_strided_store.
3415 // To calculate the high base address, we need to sum to the low base
3416 // address stride number of bytes for each element already stored by low,
3417 // that is: Ptr = Ptr + (LoEVL * Stride)
3418 EVT PtrVT = N->getBasePtr().getValueType();
3420 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
3421 DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT));
3422 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment);
3423
3424 Align Alignment = N->getOriginalAlign();
3425 if (LoMemVT.isScalableVector())
3426 Alignment = commonAlignment(Alignment,
3427 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3428
3429 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3430 MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3432 N->getAAInfo(), N->getRanges());
3433
3434 SDValue Hi = DAG.getStridedStoreVP(
3435 N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
3436 HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(),
3437 N->isCompressingStore());
3438
3439 // Build a factor node to remember that this store is independent of the
3440 // other one.
3441 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3442}
3443
3444SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3445 unsigned OpNo) {
3446 assert(N->isUnindexed() && "Indexed masked store of vector?");
3447 SDValue Ch = N->getChain();
3448 SDValue Ptr = N->getBasePtr();
3449 SDValue Offset = N->getOffset();
3450 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3451 SDValue Mask = N->getMask();
3452 SDValue Data = N->getValue();
3453 Align Alignment = N->getOriginalAlign();
3454 SDLoc DL(N);
3455
3456 SDValue DataLo, DataHi;
3457 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3458 // Split Data operand
3459 GetSplitVector(Data, DataLo, DataHi);
3460 else
3461 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3462
3463 // Split Mask operand
3464 SDValue MaskLo, MaskHi;
3465 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3466 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3467 } else {
3468 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3469 GetSplitVector(Mask, MaskLo, MaskHi);
3470 else
3471 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3472 }
3473
3474 EVT MemoryVT = N->getMemoryVT();
3475 EVT LoMemVT, HiMemVT;
3476 bool HiIsEmpty = false;
3477 std::tie(LoMemVT, HiMemVT) =
3478 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3479
3480 SDValue Lo, Hi, Res;
3481 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3482 N->getPointerInfo(), MachineMemOperand::MOStore,
3483 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3484
3485 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3486 N->getAddressingMode(), N->isTruncatingStore(),
3487 N->isCompressingStore());
3488
3489 if (HiIsEmpty) {
3490 // The hi masked store has zero storage size.
3491 // Only the lo masked store is needed.
3492 Res = Lo;
3493 } else {
3494
3495 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3496 N->isCompressingStore());
3497
3499 if (LoMemVT.isScalableVector()) {
3500 Alignment = commonAlignment(
3501 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3502 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3503 } else
3504 MPI = N->getPointerInfo().getWithOffset(
3505 LoMemVT.getStoreSize().getFixedValue());
3506
3507 MMO = DAG.getMachineFunction().getMachineMemOperand(
3509 N->getAAInfo(), N->getRanges());
3510
3511 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3512 N->getAddressingMode(), N->isTruncatingStore(),
3513 N->isCompressingStore());
3514
3515 // Build a factor node to remember that this store is independent of the
3516 // other one.
3517 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3518 }
3519
3520 return Res;
3521}
3522
3523SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3524 SDValue Ch = N->getChain();
3525 SDValue Ptr = N->getBasePtr();
3526 EVT MemoryVT = N->getMemoryVT();
3527 Align Alignment = N->getOriginalAlign();
3528 SDLoc DL(N);
3529 struct Operands {
3530 SDValue Mask;
3531 SDValue Index;
3532 SDValue Scale;
3533 SDValue Data;
3534 } Ops = [&]() -> Operands {
3535 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3536 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3537 MSC->getValue()};
3538 }
3539 auto *VPSC = cast<VPScatterSDNode>(N);
3540 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3541 VPSC->getValue()};
3542 }();
3543 // Split all operands
3544
3545 EVT LoMemVT, HiMemVT;
3546 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3547
3548 SDValue DataLo, DataHi;
3549 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3550 // Split Data operand
3551 GetSplitVector(Ops.Data, DataLo, DataHi);
3552 else
3553 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3554
3555 // Split Mask operand
3556 SDValue MaskLo, MaskHi;
3557 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3558 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3559 } else {
3560 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3561 }
3562
3563 SDValue IndexHi, IndexLo;
3564 if (getTypeAction(Ops.Index.getValueType()) ==
3566 GetSplitVector(Ops.Index, IndexLo, IndexHi);
3567 else
3568 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3569
3570 SDValue Lo;
3571 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3572 N->getPointerInfo(), MachineMemOperand::MOStore,
3573 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3574
3575 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3576 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3577 Lo =
3578 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3579 MSC->getIndexType(), MSC->isTruncatingStore());
3580
3581 // The order of the Scatter operation after split is well defined. The "Hi"
3582 // part comes after the "Lo". So these two operations should be chained one
3583 // after another.
3584 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3585 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3586 MMO, MSC->getIndexType(),
3587 MSC->isTruncatingStore());
3588 }
3589 auto *VPSC = cast<VPScatterSDNode>(N);
3590 SDValue EVLLo, EVLHi;
3591 std::tie(EVLLo, EVLHi) =
3592 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3593
3594 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3595 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3596 VPSC->getIndexType());
3597
3598 // The order of the Scatter operation after split is well defined. The "Hi"
3599 // part comes after the "Lo". So these two operations should be chained one
3600 // after another.
3601 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3602 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3603 VPSC->getIndexType());
3604}
3605
3606SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3607 assert(N->isUnindexed() && "Indexed store of vector?");
3608 assert(OpNo == 1 && "Can only split the stored value");
3609 SDLoc DL(N);
3610
3611 bool isTruncating = N->isTruncatingStore();
3612 SDValue Ch = N->getChain();
3613 SDValue Ptr = N->getBasePtr();
3614 EVT MemoryVT = N->getMemoryVT();
3615 Align Alignment = N->getOriginalAlign();
3616 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3617 AAMDNodes AAInfo = N->getAAInfo();
3618 SDValue Lo, Hi;
3619 GetSplitVector(N->getOperand(1), Lo, Hi);
3620
3621 EVT LoMemVT, HiMemVT;
3622 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3623
3624 // Scalarize if the split halves are not byte-sized.
3625 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3626 return TLI.scalarizeVectorStore(N, DAG);
3627
3628 if (isTruncating)
3629 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3630 Alignment, MMOFlags, AAInfo);
3631 else
3632 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3633 AAInfo);
3634
3636 IncrementPointer(N, LoMemVT, MPI, Ptr);
3637
3638 if (isTruncating)
3639 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3640 HiMemVT, Alignment, MMOFlags, AAInfo);
3641 else
3642 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3643
3644 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3645}
3646
3647SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3648 SDLoc DL(N);
3649
3650 // The input operands all must have the same type, and we know the result
3651 // type is valid. Convert this to a buildvector which extracts all the
3652 // input elements.
3653 // TODO: If the input elements are power-two vectors, we could convert this to
3654 // a new CONCAT_VECTORS node with elements that are half-wide.
3656 EVT EltVT = N->getValueType(0).getVectorElementType();
3657 for (const SDValue &Op : N->op_values()) {
3658 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3659 i != e; ++i) {
3660 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3661 DAG.getVectorIdxConstant(i, DL)));
3662 }
3663 }
3664
3665 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3666}
3667
3668SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3669 // The result type is legal, but the input type is illegal. If splitting
3670 // ends up with the result type of each half still being legal, just
3671 // do that. If, however, that would result in an illegal result type,
3672 // we can try to get more clever with power-two vectors. Specifically,
3673 // split the input type, but also widen the result element size, then
3674 // concatenate the halves and truncate again. For example, consider a target
3675 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3676 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3677 // %inlo = v4i32 extract_subvector %in, 0
3678 // %inhi = v4i32 extract_subvector %in, 4
3679 // %lo16 = v4i16 trunc v4i32 %inlo
3680 // %hi16 = v4i16 trunc v4i32 %inhi
3681 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3682 // %res = v8i8 trunc v8i16 %in16
3683 //
3684 // Without this transform, the original truncate would end up being
3685 // scalarized, which is pretty much always a last resort.
3686 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3687 SDValue InVec = N->getOperand(OpNo);
3688 EVT InVT = InVec->getValueType(0);
3689 EVT OutVT = N->getValueType(0);
3690 ElementCount NumElements = OutVT.getVectorElementCount();
3691 bool IsFloat = OutVT.isFloatingPoint();
3692
3693 unsigned InElementSize = InVT.getScalarSizeInBits();
3694 unsigned OutElementSize = OutVT.getScalarSizeInBits();
3695
3696 // Determine the split output VT. If its legal we can just split dirctly.
3697 EVT LoOutVT, HiOutVT;
3698 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3699 assert(LoOutVT == HiOutVT && "Unequal split?");
3700
3701 // If the input elements are only 1/2 the width of the result elements,
3702 // just use the normal splitting. Our trick only work if there's room
3703 // to split more than once.
3704 if (isTypeLegal(LoOutVT) ||
3705 InElementSize <= OutElementSize * 2)
3706 return SplitVecOp_UnaryOp(N);
3707 SDLoc DL(N);
3708
3709 // Don't touch if this will be scalarized.
3710 EVT FinalVT = InVT;
3711 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3712 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3713
3714 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3715 return SplitVecOp_UnaryOp(N);
3716
3717 // Get the split input vector.
3718 SDValue InLoVec, InHiVec;
3719 GetSplitVector(InVec, InLoVec, InHiVec);
3720
3721 // Truncate them to 1/2 the element size.
3722 //
3723 // This assumes the number of elements is a power of two; any vector that
3724 // isn't should be widened, not split.
3725 EVT HalfElementVT = IsFloat ?
3726 EVT::getFloatingPointVT(InElementSize/2) :
3727 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3728 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3729 NumElements.divideCoefficientBy(2));
3730
3731 SDValue HalfLo;
3732 SDValue HalfHi;
3733 SDValue Chain;
3734 if (N->isStrictFPOpcode()) {
3735 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3736 {N->getOperand(0), InLoVec});
3737 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3738 {N->getOperand(0), InHiVec});
3739 // Legalize the chain result - switch anything that used the old chain to
3740 // use the new one.
3741 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3742 HalfHi.getValue(1));
3743 } else {
3744 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3745 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3746 }
3747
3748 // Concatenate them to get the full intermediate truncation result.
3749 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3750 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3751 HalfHi);
3752 // Now finish up by truncating all the way down to the original result
3753 // type. This should normally be something that ends up being legal directly,
3754 // but in theory if a target has very wide vectors and an annoyingly
3755 // restricted set of legal types, this split can chain to build things up.
3756
3757 if (N->isStrictFPOpcode()) {
3758 SDValue Res = DAG.getNode(
3759 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3760 {Chain, InterVec,
3761 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3762 // Relink the chain
3763 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3764 return Res;
3765 }
3766
3767 return IsFloat
3768 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3769 DAG.getTargetConstant(
3770 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3771 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3772}
3773
3774SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3775 assert(N->getValueType(0).isVector() &&
3776 N->getOperand(0).getValueType().isVector() &&
3777 "Operand types must be vectors");
3778 // The result has a legal vector type, but the input needs splitting.
3779 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3780 SDLoc DL(N);
3781 GetSplitVector(N->getOperand(0), Lo0, Hi0);
3782 GetSplitVector(N->getOperand(1), Lo1, Hi1);
3783 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3784
3785 LLVMContext &Context = *DAG.getContext();
3786 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3787 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3788
3789 if (N->getOpcode() == ISD::SETCC) {
3790 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
3791 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
3792 } else {
3793 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3794 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3795 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
3796 std::tie(EVLLo, EVLHi) =
3797 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
3798 LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1,
3799 N->getOperand(2), MaskLo, EVLLo);
3800 HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1,
3801 N->getOperand(2), MaskHi, EVLHi);
3802 }
3803 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
3804
3805 EVT OpVT = N->getOperand(0).getValueType();
3806 ISD::NodeType ExtendCode =
3808 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
3809}
3810
3811
3812SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3813 // The result has a legal vector type, but the input needs splitting.
3814 EVT ResVT = N->getValueType(0);
3815 SDValue Lo, Hi;
3816 SDLoc DL(N);
3817 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3818 EVT InVT = Lo.getValueType();
3819
3820 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3821 InVT.getVectorElementCount());
3822
3823 if (N->isStrictFPOpcode()) {
3824 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3825 { N->getOperand(0), Lo, N->getOperand(2) });
3826 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3827 { N->getOperand(0), Hi, N->getOperand(2) });
3828 // Legalize the chain result - switch anything that used the old chain to
3829 // use the new one.
3830 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3831 Lo.getValue(1), Hi.getValue(1));
3832 ReplaceValueWith(SDValue(N, 1), NewChain);
3833 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
3834 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3835 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3836 std::tie(EVLLo, EVLHi) =
3837 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL);
3838 Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo);
3839 Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi);
3840 } else {
3841 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
3842 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
3843 }
3844
3845 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
3846}
3847
3848SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
3849 // The result (and the first input) has a legal vector type, but the second
3850 // input needs splitting.
3851
3852 SDLoc DL(N);
3853
3854 EVT LHSLoVT, LHSHiVT;
3855 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
3856
3857 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
3858 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
3859
3860 SDValue LHSLo, LHSHi;
3861 std::tie(LHSLo, LHSHi) =
3862 DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT);
3863
3864 SDValue RHSLo, RHSHi;
3865 std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL);
3866
3867 SDValue Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLoVT, LHSLo, RHSLo);
3868 SDValue Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHiVT, LHSHi, RHSHi);
3869
3870 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
3871}
3872
3873SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
3874 EVT ResVT = N->getValueType(0);
3875 SDValue Lo, Hi;
3876 SDLoc dl(N);
3877 GetSplitVector(N->getOperand(0), Lo, Hi);
3878 EVT InVT = Lo.getValueType();
3879
3880 EVT NewResVT =
3881 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3882 InVT.getVectorElementCount());
3883
3884 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
3885 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
3886
3887 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3888}
3889
3890//===----------------------------------------------------------------------===//
3891// Result Vector Widening
3892//===----------------------------------------------------------------------===//
3893
3894void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
3895 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
3896 dbgs() << "\n");
3897
3898 // See if the target wants to custom widen this node.
3899 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
3900 return;
3901
3902 SDValue Res = SDValue();
3903
3904 auto unrollExpandedOp = [&]() {
3905 // We're going to widen this vector op to a legal type by padding with undef
3906 // elements. If the wide vector op is eventually going to be expanded to
3907 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3908 // libcalls on the undef elements.
3909 EVT VT = N->getValueType(0);
3910 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3911 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3912 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3913 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3914 return true;
3915 }
3916 return false;
3917 };
3918
3919 switch (N->getOpcode()) {
3920 default:
3921#ifndef NDEBUG
3922 dbgs() << "WidenVectorResult #" << ResNo << ": ";
3923 N->dump(&DAG);
3924 dbgs() << "\n";
3925#endif
3926 llvm_unreachable("Do not know how to widen the result of this operator!");
3927
3928 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
3929 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
3930 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
3931 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
3932 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
3934 Res = WidenVecRes_INSERT_SUBVECTOR(N);
3935 break;
3936 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
3937 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
3938 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
3939 case ISD::STEP_VECTOR:
3940 case ISD::SPLAT_VECTOR:
3942 Res = WidenVecRes_ScalarOp(N);
3943 break;
3944 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
3945 case ISD::VSELECT:
3946 case ISD::SELECT:
3947 case ISD::VP_SELECT:
3948 case ISD::VP_MERGE:
3949 Res = WidenVecRes_Select(N);
3950 break;
3951 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
3952 case ISD::VP_SETCC:
3953 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3954 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3956 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3957 break;
3958 case ISD::VP_LOAD:
3959 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
3960 break;
3961 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
3962 Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N));
3963 break;
3964 case ISD::MLOAD:
3965 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3966 break;
3967 case ISD::MGATHER:
3968 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3969 break;
3970 case ISD::VP_GATHER:
3971 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
3972 break;
3974 Res = WidenVecRes_VECTOR_REVERSE(N);
3975 break;
3976
3977 case ISD::ADD: case ISD::VP_ADD:
3978 case ISD::AND: case ISD::VP_AND:
3979 case ISD::MUL: case ISD::VP_MUL:
3980 case ISD::MULHS:
3981 case ISD::MULHU:
3982 case ISD::OR: case ISD::VP_OR:
3983 case ISD::SUB: case ISD::VP_SUB:
3984 case ISD::XOR: case ISD::VP_XOR:
3985 case ISD::SHL: case ISD::VP_SHL:
3986 case ISD::SRA: case ISD::VP_ASHR:
3987 case ISD::SRL: case ISD::VP_LSHR:
3988 case ISD::FMINNUM: case ISD::VP_FMINNUM:
3989 case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
3990 case ISD::FMINIMUM:
3991 case ISD::FMAXIMUM:
3992 case ISD::SMIN: case ISD::VP_SMIN:
3993 case ISD::SMAX: case ISD::VP_SMAX:
3994 case ISD::UMIN: case ISD::VP_UMIN:
3995 case ISD::UMAX: case ISD::VP_UMAX:
3996 case ISD::UADDSAT:
3997 case ISD::SADDSAT:
3998 case ISD::USUBSAT:
3999 case ISD::SSUBSAT:
4000 case ISD::SSHLSAT:
4001 case ISD::USHLSAT:
4002 case ISD::ROTL:
4003 case ISD::ROTR:
4004 case ISD::AVGFLOORS:
4005 case ISD::AVGFLOORU:
4006 case ISD::AVGCEILS:
4007 case ISD::AVGCEILU:
4008 // Vector-predicated binary op widening. Note that -- unlike the
4009 // unpredicated versions -- we don't have to worry about trapping on
4010 // operations like UDIV, FADD, etc., as we pass on the original vector
4011 // length parameter. This means the widened elements containing garbage
4012 // aren't active.
4013 case ISD::VP_SDIV:
4014 case ISD::VP_UDIV:
4015 case ISD::VP_SREM:
4016 case ISD::VP_UREM:
4017 case ISD::VP_FADD:
4018 case ISD::VP_FSUB:
4019 case ISD::VP_FMUL:
4020 case ISD::VP_FDIV:
4021 case ISD::VP_FREM:
4022 case ISD::VP_FCOPYSIGN:
4023 Res = WidenVecRes_Binary(N);
4024 break;
4025
4026 case ISD::FPOW:
4027 case ISD::FREM:
4028 if (unrollExpandedOp())
4029 break;
4030 // If the target has custom/legal support for the scalar FP intrinsic ops
4031 // (they are probably not destined to become libcalls), then widen those
4032 // like any other binary ops.
4033 [[fallthrough]];
4034
4035 case ISD::FADD:
4036 case ISD::FMUL:
4037 case ISD::FSUB:
4038 case ISD::FDIV:
4039 case ISD::SDIV:
4040 case ISD::UDIV:
4041 case ISD::SREM:
4042 case ISD::UREM:
4043 Res = WidenVecRes_BinaryCanTrap(N);
4044 break;
4045
4046 case ISD::SMULFIX:
4047 case ISD::SMULFIXSAT:
4048 case ISD::UMULFIX:
4049 case ISD::UMULFIXSAT:
4050 // These are binary operations, but with an extra operand that shouldn't
4051 // be widened (the scale).
4052 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
4053 break;
4054
4055#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4056 case ISD::STRICT_##DAGN:
4057#include "llvm/IR/ConstrainedOps.def"
4058 Res = WidenVecRes_StrictFP(N);
4059 break;
4060
4061 case ISD::UADDO:
4062 case ISD::SADDO:
4063 case ISD::USUBO:
4064 case ISD::SSUBO:
4065 case ISD::UMULO:
4066 case ISD::SMULO:
4067 Res = WidenVecRes_OverflowOp(N, ResNo);
4068 break;
4069
4070 case ISD::FCOPYSIGN:
4071 Res = WidenVecRes_FCOPYSIGN(N);
4072 break;
4073
4074 case ISD::IS_FPCLASS:
4075 Res = WidenVecRes_IS_FPCLASS(N);
4076 break;
4077
4078 case ISD::FPOWI:
4079 Res = WidenVecRes_POWI(N);
4080 break;
4081
4085 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
4086 break;
4087
4088 case ISD::ANY_EXTEND:
4089 case ISD::FP_EXTEND:
4090 case ISD::VP_FP_EXTEND:
4091 case ISD::FP_ROUND:
4092 case ISD::VP_FP_ROUND:
4093 case ISD::FP_TO_SINT:
4094 case ISD::VP_FP_TO_SINT:
4095 case ISD::FP_TO_UINT:
4096 case ISD::VP_FP_TO_UINT:
4097 case ISD::SIGN_EXTEND:
4098 case ISD::VP_SIGN_EXTEND:
4099 case ISD::SINT_TO_FP:
4100 case ISD::VP_SINT_TO_FP:
4101 case ISD::VP_TRUNCATE:
4102 case ISD::TRUNCATE:
4103 case ISD::UINT_TO_FP:
4104 case ISD::VP_UINT_TO_FP:
4105 case ISD::ZERO_EXTEND:
4106 case ISD::VP_ZERO_EXTEND:
4107 Res = WidenVecRes_Convert(N);
4108 break;
4109
4112 Res = WidenVecRes_FP_TO_XINT_SAT(N);
4113 break;
4114
4115 case ISD::FABS:
4116 case ISD::FCEIL:
4117 case ISD::FCOS:
4118 case ISD::FEXP:
4119 case ISD::FEXP2:
4120 case ISD::FFLOOR:
4121 case ISD::FLOG:
4122 case ISD::FLOG10:
4123 case ISD::FLOG2:
4124 case ISD::FNEARBYINT:
4125 case ISD::FRINT:
4126 case ISD::FROUND:
4127 case ISD::FROUNDEVEN:
4128 case ISD::FSIN:
4129 case ISD::FSQRT:
4130 case ISD::FTRUNC:
4131 if (unrollExpandedOp())
4132 break;
4133 // If the target has custom/legal support for the scalar FP intrinsic ops
4134 // (they are probably not destined to become libcalls), then widen those
4135 // like any other unary ops.
4136 [[fallthrough]];
4137
4138 case ISD::ABS:
4139 case ISD::VP_ABS:
4140 case ISD::BITREVERSE:
4141 case ISD::VP_BITREVERSE:
4142 case ISD::BSWAP:
4143 case ISD::VP_BSWAP:
4144 case ISD::CTLZ:
4145 case ISD::VP_CTLZ:
4147 case ISD::VP_CTLZ_ZERO_UNDEF:
4148 case ISD::CTPOP:
4149