LLVM 24.0.0git
LegalizeVectorTypes.cpp
Go to the documentation of this file.
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
27#include "llvm/IR/DataLayout.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, N->getValueType(ResNo), true))
48 return;
49
50 switch (N->getOpcode()) {
51 default:
52#ifndef NDEBUG
53 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
54 N->dump(&DAG);
55 dbgs() << "\n";
56#endif
57 report_fatal_error("Do not know how to scalarize the result of this "
58 "operator!\n");
59
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
63 break;
64 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
65 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
68 R = ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(N);
69 break;
70 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
72 R = ScalarizeVecRes_VECTOR_SHUFFLE_VAR(N);
73 break;
74 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
76 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
77 break;
79 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
80 break;
81 case ISD::AssertZext:
82 case ISD::AssertSext:
83 case ISD::FPOWI:
85 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
86 break;
87 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
89 R = ScalarizeVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
90 break;
114 R = ScalarizeVecRes_ATOMIC_RMW(cast<AtomicSDNode>(N));
115 break;
116 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
117 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
120 R = ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
121 break;
122 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
123 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
124 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
125 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
126 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
128 R = ScalarizeVecRes_VECTOR_MATCH(N);
129 break;
130 case ISD::POISON:
131 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
132 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
133 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
137 R = ScalarizeVecRes_VecInregOp(N);
138 break;
139 case ISD::ABS:
141 case ISD::ANY_EXTEND:
142 case ISD::BITREVERSE:
143 case ISD::BSWAP:
144 case ISD::CTLZ:
146 case ISD::CTPOP:
147 case ISD::CTTZ:
149 case ISD::FABS:
150 case ISD::FACOS:
151 case ISD::FASIN:
152 case ISD::FATAN:
153 case ISD::FCEIL:
154 case ISD::FCOS:
155 case ISD::FCOSH:
156 case ISD::FEXP:
157 case ISD::FEXP2:
158 case ISD::FEXP10:
159 case ISD::FFLOOR:
160 case ISD::FLOG:
161 case ISD::FLOG10:
162 case ISD::FLOG2:
163 case ISD::FNEARBYINT:
164 case ISD::FNEG:
165 case ISD::FREEZE:
166 case ISD::ARITH_FENCE:
167 case ISD::FP_EXTEND:
168 case ISD::FP_TO_SINT:
169 case ISD::FP_TO_UINT:
170 case ISD::FRINT:
171 case ISD::LRINT:
172 case ISD::LLRINT:
173 case ISD::FROUND:
174 case ISD::FROUNDEVEN:
175 case ISD::LROUND:
176 case ISD::LLROUND:
177 case ISD::FSIN:
178 case ISD::FSINH:
179 case ISD::FSQRT:
180 case ISD::FTAN:
181 case ISD::FTANH:
182 case ISD::FTRUNC:
183 case ISD::SIGN_EXTEND:
184 case ISD::SINT_TO_FP:
185 case ISD::TRUNCATE:
186 case ISD::UINT_TO_FP:
187 case ISD::ZERO_EXTEND:
189 R = ScalarizeVecRes_UnaryOp(N);
190 break;
192 R = ScalarizeVecRes_ADDRSPACECAST(N);
193 break;
194 case ISD::FMODF:
195 case ISD::FFREXP:
196 case ISD::FSINCOS:
197 case ISD::FSINCOSPI:
198 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
199 break;
200 case ISD::ADD:
201 case ISD::AND:
202 case ISD::AVGCEILS:
203 case ISD::AVGCEILU:
204 case ISD::AVGFLOORS:
205 case ISD::AVGFLOORU:
206 case ISD::FADD:
207 case ISD::FCOPYSIGN:
208 case ISD::FDIV:
209 case ISD::FMUL:
210 case ISD::FMINNUM:
211 case ISD::FMAXNUM:
214 case ISD::FMINIMUM:
215 case ISD::FMAXIMUM:
216 case ISD::FMINIMUMNUM:
217 case ISD::FMAXIMUMNUM:
218 case ISD::ABDS:
219 case ISD::ABDU:
220 case ISD::SMIN:
221 case ISD::SMAX:
222 case ISD::UMIN:
223 case ISD::UMAX:
224
225 case ISD::SADDSAT:
226 case ISD::UADDSAT:
227 case ISD::SSUBSAT:
228 case ISD::USUBSAT:
229 case ISD::SSHLSAT:
230 case ISD::USHLSAT:
231
232 case ISD::FPOW:
233 case ISD::FATAN2:
234 case ISD::FREM:
235 case ISD::FSUB:
236 case ISD::MUL:
237 case ISD::MULHS:
238 case ISD::MULHU:
239 case ISD::OR:
240 case ISD::SDIV:
241 case ISD::SREM:
242 case ISD::SUB:
243 case ISD::UDIV:
244 case ISD::UREM:
245 case ISD::XOR:
246 case ISD::SHL:
247 case ISD::SRA:
248 case ISD::SRL:
249 case ISD::ROTL:
250 case ISD::ROTR:
251 case ISD::CLMUL:
252 case ISD::CLMULR:
253 case ISD::CLMULH:
254 case ISD::PEXT:
255 case ISD::PDEP:
256 R = ScalarizeVecRes_BinOp(N);
257 break;
258
259 case ISD::MASKED_UDIV:
260 case ISD::MASKED_SDIV:
261 case ISD::MASKED_UREM:
262 case ISD::MASKED_SREM:
263 R = ScalarizeVecRes_MaskedBinOp(N);
264 break;
265
266 case ISD::FLDEXP:
267 R = ScalarizeVecRes_FPOp_MultiType(N);
268 break;
269
270 case ISD::SCMP:
271 case ISD::UCMP:
272 R = ScalarizeVecRes_CMP(N);
273 break;
274
275 case ISD::FMA:
276 case ISD::FSHL:
277 case ISD::FSHR:
278 R = ScalarizeVecRes_TernaryOp(N);
279 break;
280
281#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
282 case ISD::STRICT_##DAGN:
283#include "llvm/IR/ConstrainedOps.def"
284 R = ScalarizeVecRes_StrictFPOp(N);
285 break;
286
289 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
290 break;
291
292 case ISD::UADDO:
293 case ISD::SADDO:
294 case ISD::USUBO:
295 case ISD::SSUBO:
296 case ISD::UMULO:
297 case ISD::SMULO:
298 R = ScalarizeVecRes_OverflowOp(N, ResNo);
299 break;
300 case ISD::SMULFIX:
301 case ISD::SMULFIXSAT:
302 case ISD::UMULFIX:
303 case ISD::UMULFIXSAT:
304 case ISD::SDIVFIX:
305 case ISD::SDIVFIXSAT:
306 case ISD::UDIVFIX:
307 case ISD::UDIVFIXSAT:
308 R = ScalarizeVecRes_FIX(N);
309 break;
314 R = ScalarizeVecRes_PartialReduction(N);
315 break;
316 }
317
318 // If R is null, the sub-method took care of registering the result.
319 if (R.getNode())
320 SetScalarizedVector(SDValue(N, ResNo), R);
321}
322
323SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
324 SDValue LHS = GetScalarizedVector(N->getOperand(0));
325 SDValue RHS = GetScalarizedVector(N->getOperand(1));
326 return DAG.getNode(N->getOpcode(), SDLoc(N),
327 LHS.getValueType(), LHS, RHS, N->getFlags());
328}
329
330SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
331 SDLoc DL(N);
332 SDValue LHS = GetScalarizedVector(N->getOperand(0));
333 SDValue RHS = GetScalarizedVector(N->getOperand(1));
334 SDValue Mask = N->getOperand(2);
335 EVT MaskVT = Mask.getValueType();
336 // The vselect result and input vectors need scalarizing, but it's
337 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
338 // See the similar logic in ScalarizeVecRes_SETCC.
339 if (getTypeAction(MaskVT) == TargetLowering::TypeScalarizeVector)
340 Mask = GetScalarizedVector(Mask);
341 else
342 Mask = DAG.getExtractVectorElt(DL, MaskVT.getVectorElementType(), Mask, 0);
343 // Vectors may have a different boolean contents to scalars, so truncate to i1
344 // and let type legalization promote appropriately.
345 Mask = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Mask);
346 // Masked binary ops don't have UB on disabled lanes but produce poison, so
347 // use 1 as the divisor to avoid division by zero and overflow.
348 SDValue Divisor = DAG.getSelect(DL, LHS.getValueType(), Mask, RHS,
349 DAG.getConstant(1, DL, LHS.getValueType()));
350 return DAG.getNode(ISD::getUnmaskedBinOpOpcode(N->getOpcode()), DL,
351 LHS.getValueType(), LHS, Divisor);
352}
353
354SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
355 SDLoc DL(N);
356
357 SDValue LHS = N->getOperand(0);
358 SDValue RHS = N->getOperand(1);
359 if (getTypeAction(LHS.getValueType()) ==
361 LHS = GetScalarizedVector(LHS);
362 RHS = GetScalarizedVector(RHS);
363 } else {
364 EVT VT = LHS.getValueType().getVectorElementType();
365 LHS = DAG.getExtractVectorElt(DL, VT, LHS, 0);
366 RHS = DAG.getExtractVectorElt(DL, VT, RHS, 0);
367 }
368
369 return DAG.getNode(N->getOpcode(), SDLoc(N),
370 N->getValueType(0).getVectorElementType(), LHS, RHS);
371}
372
373SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
374 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
375 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
376 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
377 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
378 Op2, N->getFlags());
379}
380
381SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
382 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
383 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
384 SDValue Op2 = N->getOperand(2);
385 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
386 Op2, N->getFlags());
387}
388
389SDValue DAGTypeLegalizer::ScalarizeVecRes_PartialReduction(SDNode *N) {
390 SDValue Acc = GetScalarizedVector(N->getOperand(0));
391 SDValue L = N->getOperand(1);
392 SDValue R = N->getOperand(2);
393 EVT ResVT = Acc.getValueType();
394 EVT InVT = L.getValueType();
395 bool IsFP = ResVT.isFloatingPoint();
396 SDLoc DL(N);
397
398 // Make sure we're not trying an extending partial reduction in this path.
399 assert(InVT.getScalarType() == ResVT && "Extending operands unsupported");
400
401 SDValue Mul = DAG.getNode(IsFP ? ISD::FMUL : ISD::MUL, DL, InVT, L, R);
402 SDValue In = DAG.getNode(IsFP ? ISD::VECREDUCE_FADD : ISD::VECREDUCE_ADD, DL,
403 ResVT, Mul);
404
405 return DAG.getNode(IsFP ? ISD::FADD : ISD::ADD, DL, ResVT, Acc, In);
406}
407
409DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
410 unsigned ResNo) {
411 assert(N->getValueType(0).getVectorNumElements() == 1 &&
412 "Unexpected vector type!");
413 SDValue Elt = GetScalarizedVector(N->getOperand(0));
414
415 EVT VT0 = N->getValueType(0);
416 EVT VT1 = N->getValueType(1);
417 SDLoc dl(N);
418
419 SDNode *ScalarNode =
420 DAG.getNode(N->getOpcode(), dl,
421 {VT0.getScalarType(), VT1.getScalarType()}, Elt)
422 .getNode();
423
424 // Replace the other vector result not being explicitly scalarized here.
425 unsigned OtherNo = 1 - ResNo;
426 EVT OtherVT = N->getValueType(OtherNo);
427 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
428 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
429 } else {
430 SDValue OtherVal = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, OtherVT,
431 SDValue(ScalarNode, OtherNo));
432 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
433 }
434
435 return SDValue(ScalarNode, ResNo);
436}
437
438SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
439 EVT VT = N->getValueType(0).getVectorElementType();
440 unsigned NumOpers = N->getNumOperands();
441 SDValue Chain = N->getOperand(0);
442 EVT ValueVTs[] = {VT, MVT::Other};
443 SDLoc dl(N);
444
445 SmallVector<SDValue, 4> Opers(NumOpers);
446
447 // The Chain is the first operand.
448 Opers[0] = Chain;
449
450 // Now process the remaining operands.
451 for (unsigned i = 1; i < NumOpers; ++i) {
452 SDValue Oper = N->getOperand(i);
453 EVT OperVT = Oper.getValueType();
454
455 if (OperVT.isVector()) {
456 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
457 Oper = GetScalarizedVector(Oper);
458 else
459 Oper =
460 DAG.getExtractVectorElt(dl, OperVT.getVectorElementType(), Oper, 0);
461 }
462
463 Opers[i] = Oper;
464 }
465
466 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
467 Opers, N->getFlags());
468
469 // Legalize the chain result - switch anything that used the old chain to
470 // use the new one.
471 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
472 return Result;
473}
474
475SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
476 unsigned ResNo) {
477 SDLoc DL(N);
478 EVT ResVT = N->getValueType(0);
479 EVT OvVT = N->getValueType(1);
480
481 SDValue ScalarLHS, ScalarRHS;
482 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
483 ScalarLHS = GetScalarizedVector(N->getOperand(0));
484 ScalarRHS = GetScalarizedVector(N->getOperand(1));
485 } else {
486 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
487 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
488 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
489 ScalarLHS = ElemsLHS[0];
490 ScalarRHS = ElemsRHS[0];
491 }
492
493 SDVTList ScalarVTs = DAG.getVTList(
495 SDNode *ScalarNode = DAG.getNode(N->getOpcode(), DL, ScalarVTs,
496 {ScalarLHS, ScalarRHS}, N->getFlags())
497 .getNode();
498
499 // Replace the other vector result not being explicitly scalarized here.
500 unsigned OtherNo = 1 - ResNo;
501 EVT OtherVT = N->getValueType(OtherNo);
502 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
503 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
504 } else {
505 SDValue OtherVal = DAG.getNode(
506 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
507 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
508 }
509
510 return SDValue(ScalarNode, ResNo);
511}
512
513SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
514 unsigned ResNo) {
515 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
516 return GetScalarizedVector(Op);
517}
518
519SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
520 SDLoc DL(N);
521 // Reuse the expansion (which should scalarize).
522 SDValue Mask = TLI.expandLoopDependenceMask(N, DAG);
523 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
524 N->getValueType(0).getScalarType(), Mask,
525 DAG.getVectorIdxConstant(0, DL));
526}
527
528SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
529 SDValue Op = N->getOperand(0);
530 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeScalarizeVector)
531 Op = GetScalarizedVector(Op);
532 EVT NewVT = N->getValueType(0).getVectorElementType();
533 return DAG.getNode(ISD::BITCAST, SDLoc(N),
534 NewVT, Op);
535}
536
537SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(SDNode *N) {
538 EVT EltVT = N->getValueType(0).getVectorElementType();
539 SDValue InOp = N->getOperand(0);
540 // The BUILD_VECTOR / SPLAT operands may be of wider element types and
541 // we may need to truncate them back to the requested return type.
542 if (EltVT.isInteger())
543 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
544 return InOp;
545}
546
547SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
548 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
549 N->getValueType(0).getVectorElementType(),
550 N->getOperand(0), N->getOperand(1));
551}
552
553SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
554 // Index 0 is the only in-range index.
555 return GetScalarizedVector(N->getOperand(0));
556}
557
558SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
559 SDLoc DL(N);
560 SDValue Op = N->getOperand(0);
561 EVT OpVT = Op.getValueType();
562 // The result needs scalarizing, but it's not a given that the source does.
563 // See similar logic in ScalarizeVecRes_UnaryOp.
564 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
565 Op = GetScalarizedVector(Op);
566 } else {
567 EVT VT = OpVT.getVectorElementType();
568 Op = DAG.getExtractVectorElt(DL, VT, Op, 0);
569 }
570 return DAG.getNode(ISD::FP_ROUND, DL,
571 N->getValueType(0).getVectorElementType(), Op,
572 N->getOperand(1));
573}
574
575SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
576 SDLoc DL(N);
577 SDValue Op = N->getOperand(0);
578 EVT OpVT = Op.getValueType();
579 // The result needs scalarizing, but it's not a given that the source does.
580 // See similar logic in ScalarizeVecRes_UnaryOp.
581 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
582 Op = GetScalarizedVector(Op);
583 } else {
584 EVT VT = OpVT.getVectorElementType();
585 Op = DAG.getExtractVectorElt(DL, VT, Op, 0);
586 }
587 return DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, DL,
588 N->getValueType(0).getVectorElementType(), Op,
589 N->getOperand(1));
590}
591
592SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
593 SDLoc DL(N);
594 SDValue Op = N->getOperand(0);
595 EVT OpVT = Op.getValueType();
596 // The result needs scalarizing, but it's not a given that the source does.
597 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
598 Op = GetScalarizedVector(Op);
599 } else {
600 EVT VT = OpVT.getVectorElementType();
601 Op = DAG.getExtractVectorElt(DL, VT, Op, 0);
602 }
603 return DAG.getNode(ISD::CONVERT_TO_ARBITRARY_FP, DL,
604 N->getValueType(0).getVectorElementType(), Op,
605 N->getOperand(1), N->getOperand(2), N->getOperand(3));
606}
607
608SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
609 SDValue Op = GetScalarizedVector(N->getOperand(0));
610 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op,
611 N->getOperand(1));
612}
613
614SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOp_MultiType(SDNode *N) {
615 SDLoc DL(N);
616 SDValue LHS = GetScalarizedVector(N->getOperand(0));
617 SDValue RHS = N->getOperand(1);
618 EVT RHSVT = RHS.getValueType();
619 // The exponent has its own type action and may not have been scalarized:
620 // v1i1 is legal on AVX-512, v1i32 is widened on AArch64.
621 if (RHSVT.isVector()) {
622 if (getTypeAction(RHSVT) == TargetLowering::TypeScalarizeVector)
623 RHS = GetScalarizedVector(RHS);
624 else
625 RHS = DAG.getExtractVectorElt(DL, RHSVT.getVectorElementType(), RHS, 0);
626 }
627 return DAG.getNode(N->getOpcode(), DL, LHS.getValueType(), LHS, RHS,
628 N->getFlags());
629}
630
631SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
632 // The value to insert may have a wider type than the vector element type,
633 // so be sure to truncate it to the element type if necessary.
634 SDValue Op = N->getOperand(1);
635 EVT EltVT = N->getValueType(0).getVectorElementType();
636 if (Op.getValueType() != EltVT)
637 // FIXME: Can this happen for floating point types?
638 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
639 return Op;
640}
641
642SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
643 SDValue Result = DAG.getAtomicLoad(
644 N->getExtensionType(), SDLoc(N), N->getMemoryVT().getVectorElementType(),
645 N->getValueType(0).getVectorElementType(), N->getChain(), N->getBasePtr(),
646 N->getMemOperand());
647
648 // Legalize the chain result - switch anything that used the old chain to
649 // use the new one.
650 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
651 return Result;
652}
653
654SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_RMW(AtomicSDNode *N) {
655 SDValue Result = DAG.getAtomic(
656 N->getOpcode(), SDLoc(N), N->getMemoryVT().getVectorElementType(),
657 N->getChain(), N->getBasePtr(), GetScalarizedVector(N->getVal()),
658 N->getMemOperand());
659
660 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
661 return Result;
662}
663
664SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
665 assert(N->isUnindexed() && "Indexed vector load?");
666
667 SDValue Result = DAG.getLoad(
668 ISD::UNINDEXED, N->getExtensionType(),
669 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
670 N->getBasePtr(), DAG.getPOISON(N->getBasePtr().getValueType()),
671 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
672 N->getBaseAlign(), N->getMemOperand()->getFlags(),
673 N->getMMOMetadataForSubAccess());
674
675 // Legalize the chain result - switch anything that used the old chain to
676 // use the new one.
677 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
678 return Result;
679}
680
681SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
682 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
683 EVT DestVT = N->getValueType(0).getVectorElementType();
684 SDValue Op = N->getOperand(0);
685 EVT OpVT = Op.getValueType();
686 SDLoc DL(N);
687 // The result needs scalarizing, but it's not a given that the source does.
688 // This is a workaround for targets where it's impossible to scalarize the
689 // result of a conversion, because the source type is legal.
690 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
691 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
692 // legal and was not scalarized.
693 // See the similar logic in ScalarizeVecRes_SETCC
694 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
695 Op = GetScalarizedVector(Op);
696 } else {
697 EVT VT = OpVT.getVectorElementType();
698 Op = DAG.getExtractVectorElt(DL, VT, Op, 0);
699 }
700 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
701}
702
703SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
704 EVT EltVT = N->getValueType(0).getVectorElementType();
705 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
706 SDValue LHS = GetScalarizedVector(N->getOperand(0));
707 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
708 LHS, DAG.getValueType(ExtVT));
709}
710
711SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
712 SDLoc DL(N);
713 SDValue Op = N->getOperand(0);
714
715 EVT OpVT = Op.getValueType();
716 EVT OpEltVT = OpVT.getVectorElementType();
717 EVT EltVT = N->getValueType(0).getVectorElementType();
718
719 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
720 Op = GetScalarizedVector(Op);
721 } else {
722 Op = DAG.getExtractVectorElt(DL, OpEltVT, Op, 0);
723 }
724
725 switch (N->getOpcode()) {
727 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
729 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
731 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
732 }
733
734 llvm_unreachable("Illegal extend_vector_inreg opcode");
735}
736
737SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
738 EVT DestVT = N->getValueType(0).getVectorElementType();
739 SDValue Op = N->getOperand(0);
740 EVT OpVT = Op.getValueType();
741 SDLoc DL(N);
742 // The result needs scalarizing, but it's not a given that the source does.
743 // This is a workaround for targets where it's impossible to scalarize the
744 // result of a conversion, because the source type is legal.
745 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
746 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
747 // legal and was not scalarized.
748 // See the similar logic in ScalarizeVecRes_SETCC
749 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
750 Op = GetScalarizedVector(Op);
751 } else {
752 EVT VT = OpVT.getVectorElementType();
753 Op = DAG.getExtractVectorElt(DL, VT, Op, 0);
754 }
755 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
756 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
757 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
758 return DAG.getAddrSpaceCast(DL, DestVT, Op, SrcAS, DestAS,
759 AddrSpaceCastN->getFlags());
760}
761
762SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
763 // If the operand is wider than the vector element type then it is implicitly
764 // truncated. Make that explicit here.
765 EVT EltVT = N->getValueType(0).getVectorElementType();
766 SDValue InOp = N->getOperand(0);
767 if (InOp.getValueType() != EltVT)
768 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
769 return InOp;
770}
771
773DAGTypeLegalizer::ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
774 assert(N->getNumValues() == N->getNumOperands() &&
775 "Expected one result per operand");
776
777 // Interleaving or deinterleaving one-element vectors leaves each result
778 // equal to the corresponding operand.
779 for (unsigned I = 0; I != N->getNumValues(); ++I)
780 SetScalarizedVector(SDValue(N, I), GetScalarizedVector(N->getOperand(I)));
781 return SDValue();
782}
783
784SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
785 SDValue Cond = N->getOperand(0);
786 EVT OpVT = Cond.getValueType();
787 SDLoc DL(N);
788 // The vselect result and true/value operands needs scalarizing, but it's
789 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
790 // See the similar logic in ScalarizeVecRes_SETCC
791 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
792 Cond = GetScalarizedVector(Cond);
793 } else {
794 EVT VT = OpVT.getVectorElementType();
795 Cond = DAG.getExtractVectorElt(DL, VT, Cond, 0);
796 }
797
798 SDValue LHS = GetScalarizedVector(N->getOperand(1));
800 TLI.getBooleanContents(false, false);
801 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
802
803 // If integer and float booleans have different contents then we can't
804 // reliably optimize in all cases. There is a full explanation for this in
805 // DAGCombiner::visitSELECT() where the same issue affects folding
806 // (select C, 0, 1) to (xor C, 1).
807 if (TLI.getBooleanContents(false, false) !=
808 TLI.getBooleanContents(false, true)) {
809 // At least try the common case where the boolean is generated by a
810 // comparison.
811 if (Cond->getOpcode() == ISD::SETCC) {
812 EVT OpVT = Cond->getOperand(0).getValueType();
813 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
814 VecBool = TLI.getBooleanContents(OpVT);
815 } else
817 }
818
819 EVT CondVT = Cond.getValueType();
820 if (ScalarBool != VecBool) {
821 switch (ScalarBool) {
823 break;
827 // Vector read from all ones, scalar expects a single 1 so mask.
828 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
829 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
830 break;
834 // Vector reads from a one, scalar from all ones so sign extend.
835 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
836 Cond, DAG.getValueType(MVT::i1));
837 break;
838 }
839 }
840
841 // Truncate the condition if needed
842 auto BoolVT = getSetCCResultType(CondVT);
843 if (BoolVT.bitsLT(CondVT))
844 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
845
846 return DAG.getSelect(SDLoc(N), LHS.getValueType(), Cond, LHS,
847 GetScalarizedVector(N->getOperand(2)), N->getFlags());
848}
849
850SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
851 SDValue LHS = GetScalarizedVector(N->getOperand(1));
852 return DAG.getSelect(SDLoc(N),
853 LHS.getValueType(), N->getOperand(0), LHS,
854 GetScalarizedVector(N->getOperand(2)));
855}
856
857SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
858 SDValue LHS = GetScalarizedVector(N->getOperand(2));
859 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
860 N->getOperand(0), N->getOperand(1),
861 LHS, GetScalarizedVector(N->getOperand(3)),
862 N->getOperand(4));
863}
864
865SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
866 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
867}
868
869SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
870 // Figure out if the scalar is the LHS or RHS and return it.
871 SDValue Arg = N->getOperand(2).getOperand(0);
872 if (Arg.isUndef())
873 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
874 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
875 return GetScalarizedVector(N->getOperand(Op));
876}
877
878SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
879 SDValue Src = N->getOperand(0);
880 EVT SrcVT = Src.getValueType();
881 SDLoc dl(N);
882
883 // Handle case where result is scalarized but operand is not
884 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
885 Src = GetScalarizedVector(Src);
886 else
887 Src = DAG.getNode(
889 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
890
891 EVT DstVT = N->getValueType(0).getVectorElementType();
892 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
893}
894
895SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
896 assert(N->getValueType(0).isVector() &&
897 N->getOperand(0).getValueType().isVector() &&
898 "Operand types must be vectors");
899 SDValue LHS = N->getOperand(0);
900 SDValue RHS = N->getOperand(1);
901 EVT OpVT = LHS.getValueType();
902 EVT NVT = N->getValueType(0).getVectorElementType();
903 SDLoc DL(N);
904
905 // The result needs scalarizing, but it's not a given that the source does.
906 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
907 LHS = GetScalarizedVector(LHS);
908 RHS = GetScalarizedVector(RHS);
909 } else {
910 EVT VT = OpVT.getVectorElementType();
911 LHS = DAG.getExtractVectorElt(DL, VT, LHS, 0);
912 RHS = DAG.getExtractVectorElt(DL, VT, RHS, 0);
913 }
914
915 // Turn it into a scalar SETCC.
916 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
917 N->getOperand(2));
918 // Vectors may have a different boolean contents to scalars. Promote the
919 // value appropriately.
920 ISD::NodeType ExtendCode =
921 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
922 return DAG.getNode(ExtendCode, DL, NVT, Res);
923}
924
925SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
926 SDLoc DL(N);
927 SDValue Arg = N->getOperand(0);
928 SDValue Test = N->getOperand(1);
929 EVT ArgVT = Arg.getValueType();
930 EVT ResultVT = N->getValueType(0).getVectorElementType();
931
932 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
933 Arg = GetScalarizedVector(Arg);
934 } else {
935 EVT VT = ArgVT.getVectorElementType();
936 Arg = DAG.getExtractVectorElt(DL, VT, Arg, 0);
937 }
938
939 SDValue Res =
940 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
941 // Vectors may have a different boolean contents to scalars. Promote the
942 // value appropriately.
943 ISD::NodeType ExtendCode =
944 TargetLowering::getExtendForContent(TLI.getBooleanContents(ArgVT));
945 return DAG.getNode(ExtendCode, DL, ResultVT, Res);
946}
947
948//===----------------------------------------------------------------------===//
949// Operand Vector Scalarization <1 x ty> -> ty.
950//===----------------------------------------------------------------------===//
951
952bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
953 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
954 N->dump(&DAG));
955 SDValue Res = SDValue();
956
957 // See if the target wants to custom scalarize this node.
958 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
959 return false;
960
961 switch (N->getOpcode()) {
962 default:
963#ifndef NDEBUG
964 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
965 N->dump(&DAG);
966 dbgs() << "\n";
967#endif
968 report_fatal_error("Do not know how to scalarize this operator's "
969 "operand!\n");
970 case ISD::BITCAST:
971 Res = ScalarizeVecOp_BITCAST(N);
972 break;
973 case ISD::FAKE_USE:
974 Res = ScalarizeVecOp_FAKE_USE(N);
975 break;
977 // Index 0 is the only in-range index.
978 Res = N->getOperand(0);
979 break;
980 case ISD::ANY_EXTEND:
981 case ISD::ZERO_EXTEND:
982 case ISD::SIGN_EXTEND:
983 case ISD::TRUNCATE:
984 case ISD::FP_TO_SINT:
985 case ISD::FP_TO_UINT:
986 case ISD::SINT_TO_FP:
987 case ISD::UINT_TO_FP:
988 case ISD::LROUND:
989 case ISD::LLROUND:
990 case ISD::LRINT:
991 case ISD::LLRINT:
992 Res = ScalarizeVecOp_UnaryOp(N);
993 break;
997 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
998 break;
1000 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1001 "Unexpected vector type!");
1002 SDValue Elt = GetScalarizedVector(N->getOperand(0));
1003 SDValue Op = DAG.getNode(
1004 N->getOpcode(), SDLoc(N), N->getValueType(0).getScalarType(), Elt,
1005 N->getOperand(1), N->getOperand(2), N->getOperand(3));
1006 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
1007 break;
1008 }
1013 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
1014 break;
1016 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
1017 break;
1019 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
1020 break;
1022 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
1023 break;
1024 case ISD::VSELECT:
1025 Res = ScalarizeVecOp_VSELECT(N);
1026 break;
1027 case ISD::SETCC:
1028 Res = ScalarizeVecOp_VSETCC(N);
1029 break;
1030 case ISD::STRICT_FSETCC:
1032 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
1033 break;
1034 case ISD::STORE:
1035 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1036 break;
1037 case ISD::ATOMIC_STORE:
1038 Res = ScalarizeVecOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1039 break;
1040 case ISD::MLOAD:
1041 Res = ScalarizeVecOp_MLOAD(cast<MaskedLoadSDNode>(N), OpNo);
1042 break;
1043 case ISD::MSTORE:
1044 Res = ScalarizeVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1045 break;
1047 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
1048 break;
1049 case ISD::FP_ROUND:
1050 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
1051 break;
1053 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
1054 break;
1055 case ISD::FP_EXTEND:
1056 Res = ScalarizeVecOp_FP_EXTEND(N);
1057 break;
1060 case ISD::VECREDUCE_ADD:
1061 case ISD::VECREDUCE_MUL:
1062 case ISD::VECREDUCE_AND:
1063 case ISD::VECREDUCE_OR:
1064 case ISD::VECREDUCE_XOR:
1075 Res = ScalarizeVecOp_VECREDUCE(N);
1076 break;
1079 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
1080 break;
1081 case ISD::SCMP:
1082 case ISD::UCMP:
1083 Res = ScalarizeVecOp_CMP(N);
1084 break;
1086 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
1087 break;
1088 case ISD::CTTZ_ELTS:
1090 Res = ScalarizeVecOp_CTTZ_ELTS(N);
1091 break;
1092 case ISD::VECTOR_MATCH:
1093 Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
1094 break;
1095 case ISD::MASKED_UDIV:
1096 case ISD::MASKED_SDIV:
1097 case ISD::MASKED_UREM:
1098 case ISD::MASKED_SREM:
1099 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
1100 break;
1101 }
1102
1103 // If the result is null, the sub-method took care of registering results etc.
1104 if (!Res.getNode()) return false;
1105
1106 // If the result is N, the sub-method updated N in place. Tell the legalizer
1107 // core about this.
1108 if (Res.getNode() == N)
1109 return true;
1110
1111 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1112 "Invalid operand expansion");
1113
1114 ReplaceValueWith(SDValue(N, 0), Res);
1115 return false;
1116}
1117
1118/// If the value to convert is a vector that needs to be scalarized, it must be
1119/// <1 x ty>. Convert the element instead.
1120SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
1121 SDValue Elt = GetScalarizedVector(N->getOperand(0));
1122 return DAG.getNode(ISD::BITCAST, SDLoc(N),
1123 N->getValueType(0), Elt);
1124}
1125
1126// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1127SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1128 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1129 "Fake Use: Unexpected vector type!");
1130 SDValue Elt = GetScalarizedVector(N->getOperand(1));
1131 return DAG.getNode(ISD::FAKE_USE, SDLoc(), MVT::Other, N->getOperand(0), Elt);
1132}
1133
1134/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1135/// Do the operation on the element instead.
1136SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1137 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1138 "Unexpected vector type!");
1139 SDValue Elt = GetScalarizedVector(N->getOperand(0));
1140 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
1141 N->getValueType(0).getScalarType(), Elt);
1142 // Revectorize the result so the types line up with what the uses of this
1143 // expression expect.
1144 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
1145}
1146
1147/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1148/// typesize).
1149SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1150 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1151 "Unexpected vector type!");
1152 SDValue Elt = GetScalarizedVector(N->getOperand(0));
1153 SDValue Op =
1154 DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0).getScalarType(),
1155 Elt, N->getOperand(1));
1156 // Revectorize the result so the types line up with what the uses of this
1157 // expression expect.
1158 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
1159}
1160
1161/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1162/// Do the strict FP operation on the element instead.
1163SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1164 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1165 "Unexpected vector type!");
1166 SDValue Elt = GetScalarizedVector(N->getOperand(1));
1167 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
1168 { N->getValueType(0).getScalarType(), MVT::Other },
1169 { N->getOperand(0), Elt });
1170 // Legalize the chain result - switch anything that used the old chain to
1171 // use the new one.
1172 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1173 // Revectorize the result so the types line up with what the uses of this
1174 // expression expect.
1175 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
1176
1177 // Do our own replacement and return SDValue() to tell the caller that we
1178 // handled all replacements since caller can only handle a single result.
1179 ReplaceValueWith(SDValue(N, 0), Res);
1180 return SDValue();
1181}
1182
1183/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1184SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1185 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1186 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1187 Ops[i] = GetScalarizedVector(N->getOperand(i));
1188 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
1189}
1190
1191/// The inserted subvector is to be scalarized - use insert vector element
1192/// instead.
1193SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1194 unsigned OpNo) {
1195 // We should not be attempting to scalarize the containing vector
1196 assert(OpNo == 1);
1197 SDValue Elt = GetScalarizedVector(N->getOperand(1));
1198 SDValue ContainingVec = N->getOperand(0);
1199 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
1200 ContainingVec.getValueType(), ContainingVec, Elt,
1201 N->getOperand(2));
1202}
1203
1204/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1205/// so just return the element, ignoring the index.
1206SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1207 EVT VT = N->getValueType(0);
1208 SDValue Res = GetScalarizedVector(N->getOperand(0));
1209 if (Res.getValueType() != VT)
1210 Res = VT.isFloatingPoint()
1211 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
1212 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
1213 return Res;
1214}
1215
1216/// If the input condition is a vector that needs to be scalarized, it must be
1217/// <1 x i1>, so just convert to a normal ISD::SELECT
1218/// (still with vector output type since that was acceptable if we got here).
1219SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1220 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
1221 EVT VT = N->getValueType(0);
1222
1223 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
1224 N->getOperand(2));
1225}
1226
1227/// If the operand is a vector that needs to be scalarized then the
1228/// result must be a single-element vector, so just convert to a scalar
1229/// SETCC and wrap with a scalar_to_vector since the res type is legal
1230/// if we got here
1231SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1232 assert(N->getValueType(0).isVector() &&
1233 N->getOperand(0).getValueType().isVector() &&
1234 "Operand types must be vectors");
1235 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1236 "Expected single-element vector type");
1237
1238 EVT VT = N->getValueType(0);
1239 SDValue LHS = GetScalarizedVector(N->getOperand(0));
1240 SDValue RHS = GetScalarizedVector(N->getOperand(1));
1241
1242 EVT OpVT = N->getOperand(0).getValueType();
1243 EVT NVT = VT.getVectorElementType();
1244 SDLoc DL(N);
1245 // Turn it into a scalar SETCC.
1246 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
1247 N->getOperand(2));
1248
1249 // Vectors may have a different boolean contents to scalars. Promote the
1250 // value appropriately.
1251 ISD::NodeType ExtendCode =
1252 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
1253
1254 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
1255
1256 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
1257}
1258
1259// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1260SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1261 unsigned OpNo) {
1262 assert(OpNo == 1 && "Wrong operand for scalarization!");
1263 assert(N->getValueType(0).isVector() &&
1264 N->getOperand(1).getValueType().isVector() &&
1265 "Operand types must be vectors");
1266 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1267 "Expected single-element vector type");
1268
1269 EVT VT = N->getValueType(0);
1270 SDValue Ch = N->getOperand(0);
1271 SDValue LHS = GetScalarizedVector(N->getOperand(1));
1272 SDValue RHS = GetScalarizedVector(N->getOperand(2));
1273 SDValue CC = N->getOperand(3);
1274
1275 EVT OpVT = N->getOperand(1).getValueType();
1276 EVT NVT = VT.getVectorElementType();
1277 SDLoc DL(N);
1278 SDValue Res = DAG.getNode(N->getOpcode(), DL, {MVT::i1, MVT::Other},
1279 {Ch, LHS, RHS, CC});
1280
1281 // Legalize the chain result - switch anything that used the old chain to
1282 // use the new one.
1283 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1284
1285 ISD::NodeType ExtendCode =
1286 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
1287
1288 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
1289 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
1290
1291 // Do our own replacement and return SDValue() to tell the caller that we
1292 // handled all replacements since caller can only handle a single result.
1293 ReplaceValueWith(SDValue(N, 0), Res);
1294 return SDValue();
1295}
1296
1297/// If the value to store is a vector that needs to be scalarized, it must be
1298/// <1 x ty>. Just store the element.
1299SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1300 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1301 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1302 SDLoc dl(N);
1303
1304 if (N->isTruncatingStore())
1305 return DAG.getTruncStore(
1306 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
1307 N->getBasePtr(), N->getPointerInfo(),
1308 N->getMemoryVT().getVectorElementType(), N->getBaseAlign(),
1309 N->getMemOperand()->getFlags(), N->getMMOMetadataForSubAccess());
1310
1311 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
1312 N->getBasePtr(), N->getPointerInfo(), N->getBaseAlign(),
1313 N->getMemOperand()->getFlags(),
1314 N->getMMOMetadataForSubAccess());
1315}
1316
1317/// If the value to store is a vector that needs to be scalarized, it must be
1318/// <1 x ty>. Just store the element.
1319SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1320 SDValue ScalarVal = GetScalarizedVector(N->getVal());
1321 return DAG.getAtomic(ISD::ATOMIC_STORE, SDLoc(N),
1322 N->getMemoryVT().getVectorElementType(), N->getChain(),
1323 ScalarVal, N->getBasePtr(), N->getMemOperand());
1324}
1325
1326/// If the mask is a vector that needs to be scalarized, it must be <1 x i1>.
1327/// Promote it to a target boolean instead.
1328SDValue DAGTypeLegalizer::ScalarizeVecOp_MLOAD(MaskedLoadSDNode *N,
1329 unsigned OpNo) {
1330 assert(OpNo == 3 && "Can only scalarize the mask operand!");
1331 EVT DataVT = N->getValueType(0);
1332 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1333 SmallVector<SDValue, 4> NewOps(N->ops());
1334 NewOps[OpNo] = Mask;
1335 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1336 if (Res == N)
1337 return SDValue(Res, 0);
1338
1339 // Update triggered CSE, do our own replacement since caller can't.
1340 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1341 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1342 return SDValue();
1343}
1344
1345/// If the mask is a vector that needs to be scalarized, it must be <1 x i1>.
1346/// Promote it to a target boolean instead.
1347SDValue DAGTypeLegalizer::ScalarizeVecOp_MSTORE(MaskedStoreSDNode *N,
1348 unsigned OpNo) {
1349 assert(!N->isCompressingStore() && "CompressingStore not expected");
1350 if (OpNo != 4)
1351 reportFatalInternalError("Scalarization of MSTORE data is not supported");
1352 EVT DataVT = N->getValue().getValueType();
1353 SmallVector<SDValue, 4> NewOps(N->ops());
1354 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1355 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1356}
1357
1358/// If the value to round is a vector that needs to be scalarized, it must be
1359/// <1 x ty>. Convert the element instead.
1360SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1361 assert(OpNo == 0 && "Wrong operand for scalarization!");
1362 SDValue Elt = GetScalarizedVector(N->getOperand(0));
1363 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1364 N->getValueType(0).getVectorElementType(), Elt,
1365 N->getOperand(1));
1366 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
1367}
1368
1369SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1370 unsigned OpNo) {
1371 assert(OpNo == 1 && "Wrong operand for scalarization!");
1372 SDValue Elt = GetScalarizedVector(N->getOperand(1));
1373 SDValue Res =
1374 DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1375 {N->getValueType(0).getVectorElementType(), MVT::Other},
1376 {N->getOperand(0), Elt, N->getOperand(2)});
1377 // Legalize the chain result - switch anything that used the old chain to
1378 // use the new one.
1379 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1380
1381 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
1382
1383 // Do our own replacement and return SDValue() to tell the caller that we
1384 // handled all replacements since caller can only handle a single result.
1385 ReplaceValueWith(SDValue(N, 0), Res);
1386 return SDValue();
1387}
1388
1389/// If the value to extend is a vector that needs to be scalarized, it must be
1390/// <1 x ty>. Convert the element instead.
1391SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1392 SDValue Elt = GetScalarizedVector(N->getOperand(0));
1393 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
1394 N->getValueType(0).getVectorElementType(), Elt);
1395 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
1396}
1397
1398/// If the value to extend is a vector that needs to be scalarized, it must be
1399/// <1 x ty>. Convert the element instead.
1400SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1401 SDValue Elt = GetScalarizedVector(N->getOperand(1));
1402 SDValue Res =
1403 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
1404 {N->getValueType(0).getVectorElementType(), MVT::Other},
1405 {N->getOperand(0), Elt});
1406 // Legalize the chain result - switch anything that used the old chain to
1407 // use the new one.
1408 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1409
1410 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
1411
1412 // Do our own replacement and return SDValue() to tell the caller that we
1413 // handled all replacements since caller can only handle a single result.
1414 ReplaceValueWith(SDValue(N, 0), Res);
1415 return SDValue();
1416}
1417
1418SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1419 SDValue Res = GetScalarizedVector(N->getOperand(0));
1420 // Result type may be wider than element type.
1421 if (Res.getValueType() != N->getValueType(0))
1422 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
1423 return Res;
1424}
1425
1426SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1427 SDValue AccOp = N->getOperand(0);
1428 SDValue VecOp = N->getOperand(1);
1429
1430 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
1431
1432 SDValue Op = GetScalarizedVector(VecOp);
1433 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
1434 AccOp, Op, N->getFlags());
1435}
1436
1437SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1438 SDValue LHS = GetScalarizedVector(N->getOperand(0));
1439 SDValue RHS = GetScalarizedVector(N->getOperand(1));
1440
1441 EVT ResVT = N->getValueType(0).getVectorElementType();
1442 SDValue Cmp = DAG.getNode(N->getOpcode(), SDLoc(N), ResVT, LHS, RHS);
1443 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Cmp);
1444}
1445
1446SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1447 // Since there is no "none-active" result, the only valid return for <1 x ty>
1448 // is 0. Note: Since we check the high mask during splitting this is safe.
1449 // As e.g., a <2 x ty> operation would split to:
1450 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1451 // : `last_active(%lo_mask)`
1452 // Which then scalarizes to:
1453 // %mask[1] ? 1 : 0
1454 EVT VT = N->getValueType(0);
1455 return DAG.getConstant(0, SDLoc(N), VT);
1456}
1457
1458SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1459 // The number of trailing zero elements is 1 if the element is 0, and 0
1460 // otherwise.
1461 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1462 return DAG.getConstant(0, SDLoc(N), N->getValueType(0));
1463 SDValue Op = GetScalarizedVector(N->getOperand(0));
1464 SDValue SetCC =
1465 DAG.getSetCC(SDLoc(N), MVT::i1, Op,
1466 DAG.getConstant(0, SDLoc(N), Op.getValueType()), ISD::SETEQ);
1467 return DAG.getZExtOrTrunc(SetCC, SDLoc(N), N->getValueType(0));
1468}
1469
1470SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
1471 SDLoc DL(N);
1472 // Reuse the expansion (which should scalarize).
1473 SDValue Mask = TLI.expandVectorMatch(N, DAG);
1474 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
1475 N->getValueType(0).getScalarType(), Mask,
1476 DAG.getVectorIdxConstant(0, DL));
1477}
1478
1479SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
1480 unsigned OpNo) {
1481 return TLI.expandVectorMatch(N, DAG);
1482}
1483
1484SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1485 assert(OpNo == 2 && "Can only scalarize mask operand");
1486 SDLoc DL(N);
1487 EVT VT = N->getOperand(0).getValueType().getVectorElementType();
1488 SDValue LHS = DAG.getExtractVectorElt(DL, VT, N->getOperand(0), 0);
1489 SDValue RHS = DAG.getExtractVectorElt(DL, VT, N->getOperand(1), 0);
1490 SDValue Mask = GetScalarizedVector(N->getOperand(2));
1491 // Vectors may have a different boolean contents to scalars, so truncate to i1
1492 // and let type legalization promote appropriately.
1493 Mask = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Mask);
1494 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1495 // use 1 as the divisor to avoid division by zero and overflow.
1496 SDValue BinOp =
1497 DAG.getNode(ISD::getUnmaskedBinOpOpcode(N->getOpcode()), DL, VT, LHS,
1498 DAG.getSelect(DL, VT, Mask, RHS, DAG.getConstant(1, DL, VT)));
1499 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, N->getValueType(0), BinOp);
1500}
1501
1502//===----------------------------------------------------------------------===//
1503// Result Vector Splitting
1504//===----------------------------------------------------------------------===//
1505
1506/// This method is called when the specified result of the specified node is
1507/// found to need vector splitting. At this point, the node may also have
1508/// invalid operands or may have other results that need legalization, we just
1509/// know that (at least) one result needs vector splitting.
1510void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1511 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1512 SDValue Lo, Hi;
1513
1514 // See if the target wants to custom expand this node.
1515 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1516 return;
1517
1518 switch (N->getOpcode()) {
1519 default:
1520#ifndef NDEBUG
1521 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1522 N->dump(&DAG);
1523 dbgs() << "\n";
1524#endif
1525 report_fatal_error("Do not know how to split the result of this "
1526 "operator!\n");
1527
1530 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1531 break;
1533 SplitVecRes_MASK_BEFOREFIRST(N, Lo, Hi);
1534 break;
1535 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1536 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1537 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1538 case ISD::VSELECT:
1539 case ISD::SELECT:
1540 case ISD::VP_MERGE: SplitRes_Select(N, Lo, Hi); break;
1541 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1542 case ISD::POISON:
1543 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1544 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1545 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1546 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1547 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1548 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1549 case ISD::FPOWI:
1550 case ISD::FLDEXP:
1551 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1552 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1553 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1554 case ISD::SPLAT_VECTOR:
1556 SplitVecRes_ScalarOp(N, Lo, Hi);
1557 break;
1558 case ISD::STEP_VECTOR:
1559 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1560 break;
1561 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1562 case ISD::ATOMIC_LOAD:
1563 SplitVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N), Lo, Hi);
1564 break;
1565 case ISD::LOAD:
1566 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
1567 break;
1568 case ISD::VP_LOAD:
1569 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
1570 break;
1571 case ISD::VP_LOAD_FF:
1572 SplitVecRes_VP_LOAD_FF(cast<VPLoadFFSDNode>(N), Lo, Hi);
1573 break;
1574 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1575 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi);
1576 break;
1577 case ISD::MLOAD:
1578 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
1579 break;
1580 case ISD::MGATHER:
1581 case ISD::VP_GATHER:
1582 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
1583 break;
1585 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1586 break;
1588 SplitVecRes_VECTOR_SHUFFLE_VAR(N, Lo, Hi);
1589 break;
1590 case ISD::SETCC:
1591 SplitVecRes_SETCC(N, Lo, Hi);
1592 break;
1593 case ISD::VECTOR_REPEAT:
1594 SplitVecRes_VECTOR_REPEAT(N, Lo, Hi);
1595 break;
1597 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1598 break;
1600 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
1601 break;
1604 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1605 break;
1607 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1608 return;
1610 SplitVecRes_VECTOR_INTERLEAVE(N);
1611 return;
1612 case ISD::VAARG:
1613 SplitVecRes_VAARG(N, Lo, Hi);
1614 break;
1615
1619 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1620 break;
1621
1622 case ISD::ABS:
1624 case ISD::BITREVERSE:
1625 case ISD::BSWAP:
1626 case ISD::CTLZ:
1627 case ISD::CTTZ:
1630 case ISD::CTPOP:
1631 case ISD::FABS:
1632 case ISD::FACOS:
1633 case ISD::FASIN:
1634 case ISD::FATAN:
1635 case ISD::FCEIL:
1636 case ISD::FCOS:
1637 case ISD::FCOSH:
1638 case ISD::FEXP:
1639 case ISD::FEXP2:
1640 case ISD::FEXP10:
1641 case ISD::FFLOOR:
1642 case ISD::FLOG:
1643 case ISD::FLOG10:
1644 case ISD::FLOG2:
1645 case ISD::FNEARBYINT:
1646 case ISD::FNEG:
1647 case ISD::FREEZE:
1648 case ISD::ARITH_FENCE:
1649 case ISD::FP_EXTEND:
1650 case ISD::FP_ROUND:
1651 case ISD::FP_TO_SINT:
1652 case ISD::FP_TO_UINT:
1653 case ISD::FRINT:
1654 case ISD::LRINT:
1655 case ISD::LLRINT:
1656 case ISD::FROUND:
1657 case ISD::FROUNDEVEN:
1658 case ISD::LROUND:
1659 case ISD::LLROUND:
1660 case ISD::FSIN:
1661 case ISD::FSINH:
1662 case ISD::FSQRT:
1663 case ISD::FTAN:
1664 case ISD::FTANH:
1665 case ISD::FTRUNC:
1666 case ISD::SINT_TO_FP:
1667 case ISD::TRUNCATE:
1668 case ISD::UINT_TO_FP:
1669 case ISD::FCANONICALIZE:
1673 SplitVecRes_UnaryOp(N, Lo, Hi);
1674 break;
1675 case ISD::ADDRSPACECAST:
1676 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1677 break;
1678 case ISD::FMODF:
1679 case ISD::FFREXP:
1680 case ISD::FSINCOS:
1681 case ISD::FSINCOSPI:
1682 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1683 break;
1684
1685 case ISD::ANY_EXTEND:
1686 case ISD::SIGN_EXTEND:
1687 case ISD::ZERO_EXTEND:
1688 SplitVecRes_ExtendOp(N, Lo, Hi);
1689 break;
1690
1691 case ISD::ADD:
1692 case ISD::SUB:
1693 case ISD::MUL:
1694 case ISD::CLMUL:
1695 case ISD::CLMULR:
1696 case ISD::CLMULH:
1697 case ISD::PEXT:
1698 case ISD::PDEP:
1699 case ISD::MULHS:
1700 case ISD::MULHU:
1701 case ISD::ABDS:
1702 case ISD::ABDU:
1703 case ISD::AVGCEILS:
1704 case ISD::AVGCEILU:
1705 case ISD::AVGFLOORS:
1706 case ISD::AVGFLOORU:
1707 case ISD::FADD:
1708 case ISD::FSUB:
1709 case ISD::FMUL:
1710 case ISD::FMINNUM:
1711 case ISD::FMINNUM_IEEE:
1712 case ISD::FMAXNUM:
1713 case ISD::FMAXNUM_IEEE:
1714 case ISD::FMINIMUM:
1715 case ISD::FMAXIMUM:
1716 case ISD::FMINIMUMNUM:
1717 case ISD::FMAXIMUMNUM:
1718 case ISD::SDIV: case ISD::VP_SDIV:
1719 case ISD::UDIV: case ISD::VP_UDIV:
1720 case ISD::FDIV:
1721 case ISD::FPOW:
1722 case ISD::FATAN2:
1723 case ISD::AND:
1724 case ISD::OR:
1725 case ISD::XOR:
1726 case ISD::SHL:
1727 case ISD::SRA:
1728 case ISD::SRL:
1729 case ISD::UREM: case ISD::VP_UREM:
1730 case ISD::SREM: case ISD::VP_SREM:
1731 case ISD::FREM:
1732 case ISD::SMIN:
1733 case ISD::SMAX:
1734 case ISD::UMIN:
1735 case ISD::UMAX:
1736 case ISD::SADDSAT:
1737 case ISD::UADDSAT:
1738 case ISD::SSUBSAT:
1739 case ISD::USUBSAT:
1740 case ISD::SSHLSAT:
1741 case ISD::USHLSAT:
1742 case ISD::ROTL:
1743 case ISD::ROTR:
1744 SplitVecRes_BinOp(N, Lo, Hi);
1745 break;
1746 case ISD::MASKED_UDIV:
1747 case ISD::MASKED_SDIV:
1748 case ISD::MASKED_UREM:
1749 case ISD::MASKED_SREM:
1750 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1751 break;
1752 case ISD::FMA:
1753 case ISD::FSHL:
1754 case ISD::FSHR:
1755 SplitVecRes_TernaryOp(N, Lo, Hi);
1756 break;
1757
1758 case ISD::SCMP: case ISD::UCMP:
1759 SplitVecRes_CMP(N, Lo, Hi);
1760 break;
1761
1762#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1763 case ISD::STRICT_##DAGN:
1764#include "llvm/IR/ConstrainedOps.def"
1765 SplitVecRes_StrictFPOp(N, Lo, Hi);
1766 break;
1767
1770 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1771 break;
1772
1773 case ISD::UADDO:
1774 case ISD::SADDO:
1775 case ISD::USUBO:
1776 case ISD::SSUBO:
1777 case ISD::UMULO:
1778 case ISD::SMULO:
1779 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1780 break;
1781 case ISD::SMULFIX:
1782 case ISD::SMULFIXSAT:
1783 case ISD::UMULFIX:
1784 case ISD::UMULFIXSAT:
1785 case ISD::SDIVFIX:
1786 case ISD::SDIVFIXSAT:
1787 case ISD::UDIVFIX:
1788 case ISD::UDIVFIXSAT:
1789 SplitVecRes_FIX(N, Lo, Hi);
1790 break;
1791 case ISD::EXPERIMENTAL_VP_SPLICE:
1792 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1793 break;
1794 case ISD::EXPERIMENTAL_VP_REVERSE:
1795 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1796 break;
1801 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1802 break;
1804 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1805 break;
1806 case ISD::VECTOR_MATCH:
1807 SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
1808 break;
1809 }
1810
1811 // If Lo/Hi is null, the sub-method took care of registering results etc.
1812 if (Lo.getNode())
1813 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1814}
1815
1816void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1817 MachinePointerInfo &MPI, SDValue &Ptr,
1818 uint64_t *ScaledOffset) {
1819 SDLoc DL(N);
1820 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1821
1822 if (MemVT.isScalableVector()) {
1823 SDValue BytesIncrement = DAG.getVScale(
1824 DL, Ptr.getValueType(),
1825 APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1826 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1827 if (ScaledOffset)
1828 *ScaledOffset += IncrementSize;
1829 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1831 } else {
1832 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1833 // Increment the pointer to the other half.
1834 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::getFixed(IncrementSize));
1835 }
1836}
1837
1838std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1839 return SplitMask(Mask, SDLoc(Mask));
1840}
1841
1842std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1843 const SDLoc &DL) {
1844 SDValue MaskLo, MaskHi;
1845 EVT MaskVT = Mask.getValueType();
1846 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1847 GetSplitVector(Mask, MaskLo, MaskHi);
1848 else
1849 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1850 return std::make_pair(MaskLo, MaskHi);
1851}
1852
1853void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1854 SDValue LHSLo, LHSHi;
1855 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1856 SDValue RHSLo, RHSHi;
1857 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1858 SDLoc dl(N);
1859
1860 const SDNodeFlags Flags = N->getFlags();
1861 unsigned Opcode = N->getOpcode();
1862 if (N->getNumOperands() == 2) {
1863 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1864 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1865 return;
1866 }
1867
1868 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1869 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
1870 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
1871 "Expected VP opcode");
1872
1873 SDValue MaskLo, MaskHi;
1874 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1875
1876 SDValue EVLLo, EVLHi;
1877 std::tie(EVLLo, EVLHi) =
1878 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1879
1880 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1881 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1882 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1883 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1884}
1885
1886void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1887 SDValue &Hi) {
1888 SDValue LHSLo, LHSHi;
1889 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1890 SDValue RHSLo, RHSHi;
1891 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1892
1893 SDValue MaskLo, MaskHi, Mask = N->getOperand(2);
1894 if (Mask.getOpcode() == ISD::SETCC)
1895 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1896 else
1897 std::tie(MaskLo, MaskHi) = SplitMask(Mask);
1898
1899 SDLoc dl(N);
1900
1901 const SDNodeFlags Flags = N->getFlags();
1902 unsigned Opcode = N->getOpcode();
1903 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, MaskLo,
1904 Flags);
1905 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, MaskHi,
1906 Flags);
1907}
1908
1909void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1910 SDValue &Hi) {
1911 SDValue Op0Lo, Op0Hi;
1912 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1913 SDValue Op1Lo, Op1Hi;
1914 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1915 SDValue Op2Lo, Op2Hi;
1916 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1917 SDLoc dl(N);
1918
1919 const SDNodeFlags Flags = N->getFlags();
1920 unsigned Opcode = N->getOpcode();
1921 Lo =
1922 DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1923 Hi =
1924 DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1925}
1926
1927void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1928 LLVMContext &Ctxt = *DAG.getContext();
1929 SDLoc dl(N);
1930
1931 SDValue LHS = N->getOperand(0);
1932 SDValue RHS = N->getOperand(1);
1933
1934 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1935 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1936 GetSplitVector(LHS, LHSLo, LHSHi);
1937 GetSplitVector(RHS, RHSLo, RHSHi);
1938 } else {
1939 std::tie(LHSLo, LHSHi) = DAG.SplitVector(LHS, dl);
1940 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, dl);
1941 }
1942
1943 EVT SplitResVT = N->getValueType(0).getHalfNumVectorElementsVT(Ctxt);
1944 Lo = DAG.getNode(N->getOpcode(), dl, SplitResVT, LHSLo, RHSLo);
1945 Hi = DAG.getNode(N->getOpcode(), dl, SplitResVT, LHSHi, RHSHi);
1946}
1947
1948void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1949 SDValue LHSLo, LHSHi;
1950 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1951 SDValue RHSLo, RHSHi;
1952 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1953 SDLoc dl(N);
1954 SDValue Op2 = N->getOperand(2);
1955
1956 unsigned Opcode = N->getOpcode();
1957 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1958 N->getFlags());
1959 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1960 N->getFlags());
1961}
1962
1963void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1964 SDValue &Hi) {
1965 // We know the result is a vector. The input may be either a vector or a
1966 // scalar value.
1967 EVT LoVT, HiVT;
1968 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1969 SDLoc dl(N);
1970
1971 SDValue InOp = N->getOperand(0);
1972 EVT InVT = InOp.getValueType();
1973
1974 // Handle some special cases efficiently.
1975 switch (getTypeAction(InVT)) {
1982 break;
1985 // A scalar to vector conversion, where the scalar needs expansion.
1986 // If the vector is being split in two then we can just convert the
1987 // expanded pieces.
1988 if (LoVT == HiVT) {
1989 GetExpandedOp(InOp, Lo, Hi);
1990 if (DAG.getDataLayout().isBigEndian())
1991 std::swap(Lo, Hi);
1992 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1993 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1994 return;
1995 }
1996 break;
1998 // If the input is a vector that needs to be split, convert each split
1999 // piece of the input now.
2000 GetSplitVector(InOp, Lo, Hi);
2001 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
2002 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
2003 return;
2005 report_fatal_error("scalarization of scalable vectors is not supported");
2006 }
2007
2008 if (LoVT.isScalableVector()) {
2009 auto [InLo, InHi] = DAG.SplitVectorOperand(N, 0);
2010 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, InLo);
2011 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, InHi);
2012 return;
2013 }
2014
2015 // In the general case, convert the input to an integer and split it by hand.
2016 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
2017 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
2018 if (DAG.getDataLayout().isBigEndian())
2019 std::swap(LoIntVT, HiIntVT);
2020
2021 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
2022
2023 if (DAG.getDataLayout().isBigEndian())
2024 std::swap(Lo, Hi);
2025 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
2026 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
2027}
2028
2029void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
2030 SDValue &Hi) {
2031 SDLoc DL(N);
2032 EVT LoVT, HiVT;
2033 SDValue PtrA = N->getOperand(0);
2034 SDValue PtrB = N->getOperand(1);
2035 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2036
2037 // The lane offset for the "Lo" half of the mask is unchanged.
2038 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, PtrA, PtrB,
2039 /*ElementSizeInBytes=*/N->getOperand(2),
2040 /*LaneOffset=*/N->getOperand(3));
2041 // The lane offset for the "Hi" half of the mask is incremented by the number
2042 // of elements in the "Lo" half.
2043 unsigned LaneOffset =
2044 N->getConstantOperandVal(3) + LoVT.getVectorMinNumElements();
2045 // Note: The lane offset is implicitly scalable for scalable masks.
2046 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, PtrA, PtrB,
2047 /*ElementSizeInBytes=*/N->getOperand(2),
2048 /*LaneOffset=*/DAG.getConstant(LaneOffset, DL, MVT::i64));
2049}
2050
2051void DAGTypeLegalizer::SplitVecRes_MASK_BEFOREFIRST(SDNode *N, SDValue &Lo,
2052 SDValue &Hi) {
2053 SDLoc DL(N);
2054 SDValue InLo, InHi;
2055 GetSplitVector(N->getOperand(0), InLo, InHi);
2056 EVT VT = InLo.getValueType();
2057 Lo = DAG.getNode(ISD::MASK_BEFOREFIRST, DL, VT, InLo);
2058
2059 // hi = AnyLoActive ? all-zeros : (mask_beforefirst hi)
2060 SDValue AnyLoActive = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, InLo);
2061 SDValue Cond = DAG.getBoolExtOrTrunc(AnyLoActive, DL,
2062 getSetCCResultType(MVT::i1), MVT::i1);
2063 Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond, DAG.getConstant(0, DL, VT),
2064 DAG.getNode(ISD::MASK_BEFOREFIRST, DL, VT, InHi));
2065}
2066
2067void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
2068 SDValue &Hi) {
2069 EVT LoVT, HiVT;
2070 SDLoc dl(N);
2071 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2072 unsigned LoNumElts = LoVT.getVectorNumElements();
2073 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
2074 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
2075
2076 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
2077 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
2078}
2079
2080void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
2081 SDValue &Hi) {
2082 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
2083 SDLoc dl(N);
2084 unsigned NumSubvectors = N->getNumOperands() / 2;
2085 if (NumSubvectors == 1) {
2086 Lo = N->getOperand(0);
2087 Hi = N->getOperand(1);
2088 return;
2089 }
2090
2091 EVT LoVT, HiVT;
2092 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2093
2094 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
2095 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
2096
2097 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
2098 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
2099}
2100
2101void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
2102 SDValue &Hi) {
2103 SDValue Vec = N->getOperand(0);
2104 SDValue Idx = N->getOperand(1);
2105 SDLoc dl(N);
2106
2107 EVT LoVT, HiVT;
2108 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2109
2110 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
2111 uint64_t IdxVal = Idx->getAsZExtVal();
2112 Hi = DAG.getNode(
2113 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
2114 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
2115}
2116
2117void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
2118 SDValue &Hi) {
2119 SDValue Vec = N->getOperand(0);
2120 SDValue SubVec = N->getOperand(1);
2121 SDValue Idx = N->getOperand(2);
2122 SDLoc dl(N);
2123 GetSplitVector(Vec, Lo, Hi);
2124
2125 EVT VecVT = Vec.getValueType();
2126 EVT LoVT = Lo.getValueType();
2127 EVT SubVecVT = SubVec.getValueType();
2128 unsigned VecElems = VecVT.getVectorMinNumElements();
2129 unsigned SubElems = SubVecVT.getVectorMinNumElements();
2130 unsigned LoElems = LoVT.getVectorMinNumElements();
2131
2132 // If we know the index is in the first half, and we know the subvector
2133 // doesn't cross the boundary between the halves, we can avoid spilling the
2134 // vector, and insert into the lower half of the split vector directly.
2135 unsigned IdxVal = Idx->getAsZExtVal();
2136 if (IdxVal + SubElems <= LoElems) {
2137 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
2138 return;
2139 }
2140 // Similarly if the subvector is fully in the high half, but mind that we
2141 // can't tell whether a fixed-length subvector is fully within the high half
2142 // of a scalable vector.
2143 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
2144 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
2145 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
2146 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
2147 return;
2148 }
2149
2150 if (getTypeAction(SubVecVT) == TargetLowering::TypeWidenVector &&
2151 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
2152 SDValue WideSubVec = GetWidenedVector(SubVec);
2153 if (WideSubVec.getValueType() == VecVT) {
2154 std::tie(Lo, Hi) = DAG.SplitVector(WideSubVec, SDLoc(WideSubVec));
2155 return;
2156 }
2157 }
2158
2159 // Spill the vector to the stack.
2160 // In cases where the vector is illegal it will be broken down into parts
2161 // and stored in parts - we should use the alignment for the smallest part.
2162 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2164 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2165 auto &MF = DAG.getMachineFunction();
2166 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2167 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2168
2169 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2170 SmallestAlign);
2171
2172 // Store the new subvector into the specified index.
2173 SDValue SubVecPtr =
2174 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
2175 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
2177
2178 // Load the Lo part from the stack slot.
2179 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
2180 SmallestAlign);
2181
2182 // Increment the pointer to the other part.
2183 auto *Load = cast<LoadSDNode>(Lo);
2184 MachinePointerInfo MPI = Load->getPointerInfo();
2185 IncrementPointer(Load, LoVT, MPI, StackPtr);
2186
2187 // Load the Hi part from the stack slot.
2188 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
2189}
2190
2191// Handle splitting an FP where the second operand does not match the first
2192// type. The second operand may be a scalar, or a vector that has exactly as
2193// many elements as the first
2194void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2195 SDValue &Hi) {
2196 SDValue LHSLo, LHSHi;
2197 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
2198 SDLoc DL(N);
2199
2200 SDValue RHSLo, RHSHi;
2201 SDValue RHS = N->getOperand(1);
2202 EVT RHSVT = RHS.getValueType();
2203 if (RHSVT.isVector()) {
2204 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
2205 GetSplitVector(RHS, RHSLo, RHSHi);
2206 else
2207 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
2208
2209 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHSLo);
2210 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHSHi);
2211 } else {
2212 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHS);
2213 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHS);
2214 }
2215}
2216
2217void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2218 SDValue &Hi) {
2219 SDLoc DL(N);
2220 SDValue ArgLo, ArgHi;
2221 SDValue Test = N->getOperand(1);
2222 SDValue FpValue = N->getOperand(0);
2223 if (getTypeAction(FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2224 GetSplitVector(FpValue, ArgLo, ArgHi);
2225 else
2226 std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue));
2227 EVT LoVT, HiVT;
2228 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2229
2230 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
2231 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
2232}
2233
2234void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2235 SDValue &Hi) {
2236 SDValue LHSLo, LHSHi;
2237 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
2238 SDLoc dl(N);
2239
2240 EVT LoVT, HiVT;
2241 std::tie(LoVT, HiVT) =
2242 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
2243
2244 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
2245 DAG.getValueType(LoVT));
2246 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
2247 DAG.getValueType(HiVT));
2248}
2249
2250void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2251 SDValue &Hi) {
2252 unsigned Opcode = N->getOpcode();
2253 SDValue N0 = N->getOperand(0);
2254
2255 SDLoc dl(N);
2256 SDValue InLo, InHi;
2257
2258 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
2259 GetSplitVector(N0, InLo, InHi);
2260 else
2261 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
2262
2263 EVT InLoVT = InLo.getValueType();
2264 unsigned InNumElements = InLoVT.getVectorNumElements();
2265
2266 EVT OutLoVT, OutHiVT;
2267 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2268 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2269 assert((2 * OutNumElements) <= InNumElements &&
2270 "Illegal extend vector in reg split");
2271
2272 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2273 // input vector (i.e. we only use InLo):
2274 // OutLo will extend the first OutNumElements from InLo.
2275 // OutHi will extend the next OutNumElements from InLo.
2276
2277 // Shuffle the elements from InLo for OutHi into the bottom elements to
2278 // create a 'fake' InHi.
2279 SmallVector<int, 8> SplitHi(InNumElements, -1);
2280 for (unsigned i = 0; i != OutNumElements; ++i)
2281 SplitHi[i] = i + OutNumElements;
2282 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getPOISON(InLoVT), SplitHi);
2283
2284 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
2285 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
2286}
2287
2288void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2289 SDValue &Hi) {
2290 unsigned NumOps = N->getNumOperands();
2291 SDValue Chain = N->getOperand(0);
2292 EVT LoVT, HiVT;
2293 SDLoc dl(N);
2294 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2295
2298
2299 // The Chain is the first operand.
2300 OpsLo[0] = Chain;
2301 OpsHi[0] = Chain;
2302
2303 // Now process the remaining operands.
2304 for (unsigned i = 1; i < NumOps; ++i) {
2305 SDValue Op = N->getOperand(i);
2306 SDValue OpLo = Op;
2307 SDValue OpHi = Op;
2308
2309 EVT InVT = Op.getValueType();
2310 if (InVT.isVector()) {
2311 // If the input also splits, handle it directly for a
2312 // compile time speedup. Otherwise split it by hand.
2313 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2314 GetSplitVector(Op, OpLo, OpHi);
2315 else
2316 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
2317 }
2318
2319 OpsLo[i] = OpLo;
2320 OpsHi[i] = OpHi;
2321 }
2322
2323 EVT LoValueVTs[] = {LoVT, MVT::Other};
2324 EVT HiValueVTs[] = {HiVT, MVT::Other};
2325 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
2326 N->getFlags());
2327 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
2328 N->getFlags());
2329
2330 // Build a factor node to remember that this Op is independent of the
2331 // other one.
2332 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2333 Lo.getValue(1), Hi.getValue(1));
2334
2335 // Legalize the chain result - switch anything that used the old chain to
2336 // use the new one.
2337 ReplaceValueWith(SDValue(N, 1), Chain);
2338}
2339
2340SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2341 EVT VT = N->getValueType(0);
2342 if (VT.isScalableVector())
2343 report_fatal_error("Cannot unroll scalable vector strict-fp operation!");
2344
2345 SDValue Chain = N->getOperand(0);
2346 unsigned NE = VT.getVectorNumElements();
2347 EVT EltVT = VT.getVectorElementType();
2348 SDLoc dl(N);
2349
2351 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2352
2353 // If ResNE is 0, fully unroll the vector op.
2354 if (ResNE == 0)
2355 ResNE = NE;
2356 else if (NE > ResNE)
2357 NE = ResNE;
2358
2359 //The results of each unrolled operation, including the chain.
2360 SDVTList ChainVTs = DAG.getVTList(EltVT, MVT::Other);
2362
2363 unsigned i;
2364 for (i = 0; i != NE; ++i) {
2365 Operands[0] = Chain;
2366 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2367 SDValue Operand = N->getOperand(j);
2368 EVT OperandVT = Operand.getValueType();
2369 if (OperandVT.isVector()) {
2370 EVT OperandEltVT = OperandVT.getVectorElementType();
2371 Operands[j] = DAG.getExtractVectorElt(dl, OperandEltVT, Operand, i);
2372 } else {
2373 Operands[j] = Operand;
2374 }
2375 }
2376 SDValue Scalar =
2377 DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands, N->getFlags());
2378
2379 //Add in the scalar as well as its chain value to the
2380 //result vectors.
2381 Scalars.push_back(Scalar);
2382 Chains.push_back(Scalar.getValue(1));
2383 }
2384
2385 for (; i < ResNE; ++i)
2386 Scalars.push_back(DAG.getPOISON(EltVT));
2387
2388 // Build a new factor node to connect the chain back together.
2389 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
2390 ReplaceValueWith(SDValue(N, 1), Chain);
2391
2392 // Create a new BUILD_VECTOR node
2393 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
2394 return DAG.getBuildVector(VecVT, dl, Scalars);
2395}
2396
2397void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2398 SDValue &Lo, SDValue &Hi) {
2399 SDLoc dl(N);
2400 EVT ResVT = N->getValueType(0);
2401 EVT OvVT = N->getValueType(1);
2402 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2403 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
2404 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
2405
2406 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2407 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
2408 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
2409 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
2410 } else {
2411 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
2412 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
2413 }
2414
2415 unsigned Opcode = N->getOpcode();
2416 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
2417 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
2418 SDNode *LoNode =
2419 DAG.getNode(Opcode, dl, LoVTs, {LoLHS, LoRHS}, N->getFlags()).getNode();
2420 SDNode *HiNode =
2421 DAG.getNode(Opcode, dl, HiVTs, {HiLHS, HiRHS}, N->getFlags()).getNode();
2422
2423 Lo = SDValue(LoNode, ResNo);
2424 Hi = SDValue(HiNode, ResNo);
2425
2426 // Replace the other vector result not being explicitly split here.
2427 unsigned OtherNo = 1 - ResNo;
2428 EVT OtherVT = N->getValueType(OtherNo);
2429 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
2430 SetSplitVector(SDValue(N, OtherNo),
2431 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
2432 } else {
2433 SDValue OtherVal = DAG.getNode(
2434 ISD::CONCAT_VECTORS, dl, OtherVT,
2435 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
2436 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
2437 }
2438}
2439
2440void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2441 SDValue &Hi) {
2442 SDValue Vec = N->getOperand(0);
2443 SDValue Elt = N->getOperand(1);
2444 SDValue Idx = N->getOperand(2);
2445 SDLoc dl(N);
2446 GetSplitVector(Vec, Lo, Hi);
2447
2448 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
2449 unsigned IdxVal = CIdx->getZExtValue();
2450 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2451 if (IdxVal < LoNumElts) {
2452 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
2453 Lo.getValueType(), Lo, Elt, Idx);
2454 return;
2455 } else if (!Vec.getValueType().isScalableVector()) {
2456 Hi = DAG.getInsertVectorElt(dl, Hi, Elt, IdxVal - LoNumElts);
2457 return;
2458 }
2459 }
2460
2461 // Make the vector elements byte-addressable if they aren't already.
2462 EVT VecVT = Vec.getValueType();
2463 EVT EltVT = VecVT.getVectorElementType();
2464 if (!EltVT.isByteSized()) {
2465 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(*DAG.getContext());
2466 VecVT = VecVT.changeElementType(*DAG.getContext(), EltVT);
2467 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2468 // Extend the element type to match if needed.
2469 if (EltVT.bitsGT(Elt.getValueType()))
2470 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
2471 }
2472
2473 // Spill the vector to the stack.
2474 // In cases where the vector is illegal it will be broken down into parts
2475 // and stored in parts - we should use the alignment for the smallest part.
2476 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2478 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2479 auto &MF = DAG.getMachineFunction();
2480 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2481 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2482
2483 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2484 SmallestAlign);
2485
2486 // Store the new element. This may be larger than the vector element type,
2487 // so use a truncating store.
2488 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2489 Store = DAG.getTruncStore(
2490 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
2491 commonAlignment(SmallestAlign,
2492 EltVT.getFixedSizeInBits() / 8));
2493
2494 EVT LoVT, HiVT;
2495 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
2496
2497 // Load the Lo part from the stack slot.
2498 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
2499
2500 // Increment the pointer to the other part.
2501 auto Load = cast<LoadSDNode>(Lo);
2502 MachinePointerInfo MPI = Load->getPointerInfo();
2503 IncrementPointer(Load, LoVT, MPI, StackPtr);
2504
2505 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
2506
2507 // If we adjusted the original type, we need to truncate the results.
2508 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2509 if (LoVT != Lo.getValueType())
2510 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
2511 if (HiVT != Hi.getValueType())
2512 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
2513}
2514
2515void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2516 SDValue &Hi) {
2517 EVT LoVT, HiVT;
2518 SDLoc dl(N);
2519 assert(N->getValueType(0).isScalableVector() &&
2520 "Only scalable vectors are supported for STEP_VECTOR");
2521 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2522 SDValue Step = N->getOperand(0);
2523
2524 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
2525
2526 // Hi = Lo + (EltCnt * Step)
2527 EVT EltVT = Step.getValueType();
2528 APInt StepVal = Step->getAsAPIntVal();
2529 SDValue StartOfHi =
2530 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
2531 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
2532 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
2533
2534 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
2535 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
2536}
2537
2538void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2539 SDValue &Hi) {
2540 EVT LoVT, HiVT;
2541 SDLoc dl(N);
2542 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2543 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
2544 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2545 Hi = DAG.getPOISON(HiVT);
2546 } else {
2547 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2548 Hi = Lo;
2549 }
2550}
2551
2552void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2553 SDValue &Hi) {
2554 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2555 "Extended load during type legalization!");
2556 SDLoc dl(LD);
2557 EVT VT = LD->getValueType(0);
2558 EVT LoVT, HiVT;
2559 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2560
2561 SDValue Ch = LD->getChain();
2562 SDValue Ptr = LD->getBasePtr();
2563
2564 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2565 EVT MemIntVT =
2566 EVT::getIntegerVT(*DAG.getContext(), LD->getMemoryVT().getSizeInBits());
2567 SDValue ALD = DAG.getAtomicLoad(LD->getExtensionType(), dl, MemIntVT, IntVT,
2568 Ch, Ptr, LD->getMemOperand());
2569
2570 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
2571 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
2572 SDValue ExtractLo, ExtractHi;
2573 SplitInteger(ALD, LoIntVT, HiIntVT, ExtractLo, ExtractHi);
2574
2575 Lo = DAG.getBitcast(LoVT, ExtractLo);
2576 Hi = DAG.getBitcast(HiVT, ExtractHi);
2577
2578 // Legalize the chain result - switch anything that used the old chain to
2579 // use the new one.
2580 ReplaceValueWith(SDValue(LD, 1), ALD.getValue(1));
2581}
2582
2583void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2584 SDValue &Hi) {
2585 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2586 EVT LoVT, HiVT;
2587 SDLoc dl(LD);
2588 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
2589
2590 ISD::LoadExtType ExtType = LD->getExtensionType();
2591 SDValue Ch = LD->getChain();
2592 SDValue Ptr = LD->getBasePtr();
2593 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
2594 EVT MemoryVT = LD->getMemoryVT();
2595 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2596 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
2597
2598 EVT LoMemVT, HiMemVT;
2599 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2600
2601 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2602 SDValue Value, NewChain;
2603 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2604 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
2605 ReplaceValueWith(SDValue(LD, 1), NewChain);
2606 return;
2607 }
2608
2609 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
2610 LD->getPointerInfo(), LoMemVT, LD->getBaseAlign(), MMOFlags,
2611 Metadata);
2612
2613 MachinePointerInfo MPI;
2614 IncrementPointer(LD, LoMemVT, MPI, Ptr);
2615
2616 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
2617 HiMemVT, LD->getBaseAlign(), MMOFlags, Metadata);
2618
2619 // Build a factor node to remember that this load is independent of the
2620 // other one.
2621 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2622 Hi.getValue(1));
2623
2624 // Legalize the chain result - switch anything that used the old chain to
2625 // use the new one.
2626 ReplaceValueWith(SDValue(LD, 1), Ch);
2627}
2628
2629void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2630 SDValue &Hi) {
2631 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2632 EVT LoVT, HiVT;
2633 SDLoc dl(LD);
2634 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
2635
2636 ISD::LoadExtType ExtType = LD->getExtensionType();
2637 SDValue Ch = LD->getChain();
2638 SDValue Ptr = LD->getBasePtr();
2639 SDValue Offset = LD->getOffset();
2640 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2641 Align Alignment = LD->getBaseAlign();
2642 SDValue Mask = LD->getMask();
2643 SDValue EVL = LD->getVectorLength();
2644 EVT MemoryVT = LD->getMemoryVT();
2645
2646 EVT LoMemVT, HiMemVT;
2647 bool HiIsEmpty = false;
2648 std::tie(LoMemVT, HiMemVT) =
2649 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2650
2651 // Split Mask operand
2652 SDValue MaskLo, MaskHi;
2653 if (Mask.getOpcode() == ISD::SETCC) {
2654 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2655 } else {
2656 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2657 GetSplitVector(Mask, MaskLo, MaskHi);
2658 else
2659 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2660 }
2661
2662 // Split EVL operand
2663 SDValue EVLLo, EVLHi;
2664 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
2665
2666 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2667 LD->getPointerInfo(), MachineMemOperand::MOLoad,
2669 MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2670
2671 Lo =
2672 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
2673 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
2674
2675 if (HiIsEmpty) {
2676 // The hi vp_load has zero storage size. We therefore simply set it to
2677 // the low vp_load and rely on subsequent removal from the chain.
2678 Hi = Lo;
2679 } else {
2680 // Generate hi vp_load.
2681 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2682 LD->isExpandingLoad());
2683
2684 MachinePointerInfo MPI;
2685 if (LoMemVT.isScalableVector())
2686 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2687 else
2688 MPI = LD->getPointerInfo().getWithOffset(
2689 LoMemVT.getStoreSize().getFixedValue());
2690
2691 MMO = DAG.getMachineFunction().getMachineMemOperand(
2693 Alignment, MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2694
2695 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
2696 Offset, MaskHi, EVLHi, HiMemVT, MMO,
2697 LD->isExpandingLoad());
2698 }
2699
2700 // Build a factor node to remember that this load is independent of the
2701 // other one.
2702 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2703 Hi.getValue(1));
2704
2705 // Legalize the chain result - switch anything that used the old chain to
2706 // use the new one.
2707 ReplaceValueWith(SDValue(LD, 1), Ch);
2708}
2709
2710void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2711 SDValue &Hi) {
2712 SDLoc dl(LD);
2713 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(LD->getValueType(0));
2714
2715 SDValue Ch = LD->getChain();
2716 SDValue Ptr = LD->getBasePtr();
2717 Align Alignment = LD->getBaseAlign();
2718 SDValue Mask = LD->getMask();
2719 SDValue EVL = LD->getVectorLength();
2720
2721 // Split Mask operand
2722 SDValue MaskLo, MaskHi;
2723 if (Mask.getOpcode() == ISD::SETCC) {
2724 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2725 } else {
2726 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2727 GetSplitVector(Mask, MaskLo, MaskHi);
2728 else
2729 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2730 }
2731
2732 // Split EVL operand
2733 auto [EVLLo, EVLHi] = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
2734
2735 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2736 LD->getPointerInfo(), MachineMemOperand::MOLoad,
2738 MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2739
2740 Lo = DAG.getLoadFFVP(LoVT, dl, Ch, Ptr, MaskLo, EVLLo, MMO);
2741
2742 // Fill the upper half with poison.
2743 Hi = DAG.getPOISON(HiVT);
2744
2745 ReplaceValueWith(SDValue(LD, 1), Lo.getValue(1));
2746 ReplaceValueWith(SDValue(LD, 2), Lo.getValue(2));
2747}
2748
2749void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2750 SDValue &Lo, SDValue &Hi) {
2751 assert(SLD->isUnindexed() &&
2752 "Indexed VP strided load during type legalization!");
2753 assert(SLD->getOffset().isUndef() &&
2754 "Unexpected indexed variable-length load offset");
2755
2756 SDLoc DL(SLD);
2757
2758 EVT LoVT, HiVT;
2759 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0));
2760
2761 EVT LoMemVT, HiMemVT;
2762 bool HiIsEmpty = false;
2763 std::tie(LoMemVT, HiMemVT) =
2764 DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty);
2765
2766 SDValue Mask = SLD->getMask();
2767 SDValue LoMask, HiMask;
2768 if (Mask.getOpcode() == ISD::SETCC) {
2769 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
2770 } else {
2771 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2772 GetSplitVector(Mask, LoMask, HiMask);
2773 else
2774 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2775 }
2776
2777 SDValue LoEVL, HiEVL;
2778 std::tie(LoEVL, HiEVL) =
2779 DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL);
2780
2781 // Generate the low vp_strided_load
2782 Lo = DAG.getStridedLoadVP(
2783 SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL,
2784 SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(),
2785 LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad());
2786
2787 if (HiIsEmpty) {
2788 // The high vp_strided_load has zero storage size. We therefore simply set
2789 // it to the low vp_strided_load and rely on subsequent removal from the
2790 // chain.
2791 Hi = Lo;
2792 } else {
2793 // Generate the high vp_strided_load.
2794 // To calculate the high base address, we need to sum to the low base
2795 // address stride number of bytes for each element already loaded by low,
2796 // that is: Ptr = Ptr + (LoEVL * Stride)
2797 EVT PtrVT = SLD->getBasePtr().getValueType();
2799 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
2800 DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT));
2801 SDValue Ptr =
2802 DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment);
2803
2804 Align Alignment = SLD->getBaseAlign();
2805 if (LoMemVT.isScalableVector())
2807 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2808
2809 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2810 MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2812 Alignment, MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
2813
2814 Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
2815 HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
2816 SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO,
2817 SLD->isExpandingLoad());
2818 }
2819
2820 // Build a factor node to remember that this load is independent of the
2821 // other one.
2822 SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
2823 Hi.getValue(1));
2824
2825 // Legalize the chain result - switch anything that used the old chain to
2826 // use the new one.
2827 ReplaceValueWith(SDValue(SLD, 1), Ch);
2828}
2829
2830void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2831 SDValue &Lo, SDValue &Hi) {
2832 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2833 EVT LoVT, HiVT;
2834 SDLoc dl(MLD);
2835 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
2836
2837 SDValue Ch = MLD->getChain();
2838 SDValue Ptr = MLD->getBasePtr();
2839 SDValue Offset = MLD->getOffset();
2840 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2841 SDValue Mask = MLD->getMask();
2842 SDValue PassThru = MLD->getPassThru();
2843 Align Alignment = MLD->getBaseAlign();
2844 ISD::LoadExtType ExtType = MLD->getExtensionType();
2845 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2846
2847 // Split Mask operand
2848 SDValue MaskLo, MaskHi;
2849 if (Mask.getOpcode() == ISD::SETCC) {
2850 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2851 } else {
2852 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2853 GetSplitVector(Mask, MaskLo, MaskHi);
2854 else
2855 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2856 }
2857
2858 EVT MemoryVT = MLD->getMemoryVT();
2859 EVT LoMemVT, HiMemVT;
2860 bool HiIsEmpty = false;
2861 std::tie(LoMemVT, HiMemVT) =
2862 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2863
2864 SDValue PassThruLo, PassThruHi;
2865 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2866 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2867 else
2868 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2869
2870 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2872 Alignment,
2873 MMOMetadata(MLD->getAAInfo(), MLD->getRanges(), MLD->getMemCacheHint()));
2874
2875 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
2876 MMO, MLD->getAddressingMode(), ExtType,
2877 MLD->isExpandingLoad());
2878
2879 if (HiIsEmpty) {
2880 // The hi masked load has zero storage size. We therefore simply set it to
2881 // the low masked load and rely on subsequent removal from the chain.
2882 Hi = Lo;
2883 } else {
2884 // Generate hi masked load.
2885 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2886 MLD->isExpandingLoad());
2887
2888 MachinePointerInfo MPI;
2889 if (LoMemVT.isScalableVector())
2890 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2891 else
2892 MPI = MLD->getPointerInfo().getWithOffset(
2893 LoMemVT.getStoreSize().getFixedValue());
2894
2895 MMO = DAG.getMachineFunction().getMachineMemOperand(
2896 MPI, MMOFlags, LocationSize::beforeOrAfterPointer(), Alignment,
2897 MMOMetadata(MLD->getAAInfo(), MLD->getRanges(),
2898 MLD->getMemCacheHint()));
2899
2900 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
2901 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
2902 MLD->isExpandingLoad());
2903 }
2904
2905 // Build a factor node to remember that this load is independent of the
2906 // other one.
2907 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2908 Hi.getValue(1));
2909
2910 // Legalize the chain result - switch anything that used the old chain to
2911 // use the new one.
2912 ReplaceValueWith(SDValue(MLD, 1), Ch);
2913
2914}
2915
2916void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2917 SDValue &Hi, bool SplitSETCC) {
2918 EVT LoVT, HiVT;
2919 SDLoc dl(N);
2920 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2921
2922 SDValue Ch = N->getChain();
2923 SDValue Ptr = N->getBasePtr();
2924 struct Operands {
2925 SDValue Mask;
2926 SDValue Index;
2927 SDValue Scale;
2928 } Ops = [&]() -> Operands {
2929 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2930 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2931 }
2932 auto *VPSC = cast<VPGatherSDNode>(N);
2933 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2934 }();
2935
2936 EVT MemoryVT = N->getMemoryVT();
2937 Align Alignment = N->getBaseAlign();
2938
2939 // Split Mask operand
2940 SDValue MaskLo, MaskHi;
2941 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2942 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2943 } else {
2944 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2945 }
2946
2947 EVT LoMemVT, HiMemVT;
2948 // Split MemoryVT
2949 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2950
2951 SDValue IndexHi, IndexLo;
2952 if (getTypeAction(Ops.Index.getValueType()) ==
2954 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2955 else
2956 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2957
2958 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2959 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2960 N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
2961 Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
2962
2963 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2964 SDValue PassThru = MGT->getPassThru();
2965 SDValue PassThruLo, PassThruHi;
2966 if (getTypeAction(PassThru.getValueType()) ==
2968 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2969 else
2970 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2971
2972 ISD::LoadExtType ExtType = MGT->getExtensionType();
2973 ISD::MemIndexType IndexTy = MGT->getIndexType();
2974
2975 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2976 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2977 OpsLo, MMO, IndexTy, ExtType);
2978
2979 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2980 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2981 OpsHi, MMO, IndexTy, ExtType);
2982 } else {
2983 auto *VPGT = cast<VPGatherSDNode>(N);
2984 SDValue EVLLo, EVLHi;
2985 std::tie(EVLLo, EVLHi) =
2986 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2987
2988 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2989 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2990 MMO, VPGT->getIndexType());
2991
2992 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2993 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2994 MMO, VPGT->getIndexType());
2995 }
2996
2997 // Build a factor node to remember that this load is independent of the
2998 // other one.
2999 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3000 Hi.getValue(1));
3001
3002 // Legalize the chain result - switch anything that used the old chain to
3003 // use the new one.
3004 ReplaceValueWith(SDValue(N, 1), Ch);
3005}
3006
3007void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
3008 SDValue &Hi) {
3009 // This is not "trivial", as there is a dependency between the two subvectors.
3010 // Depending on the number of 1s in the mask, the elements from the Hi vector
3011 // need to be moved to the Lo vector. Passthru values make this even harder.
3012 // We try to use VECTOR_COMPRESS if the target has custom lowering with
3013 // smaller types and passthru is undef, as it is most likely faster than the
3014 // fully expand path. Otherwise, just do the full expansion as one "big"
3015 // operation and then extract the Lo and Hi vectors from that. This gets
3016 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
3017 SDLoc DL(N);
3018 EVT VecVT = N->getValueType(0);
3019
3020 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VecVT);
3021 bool HasCustomLowering = false;
3022 EVT CheckVT = LoVT;
3023 while (CheckVT.getVectorMinNumElements() > 1) {
3024 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
3025 // custom lowering for illegal types. So we do the checks separately.
3026 if (TLI.isOperationLegal(ISD::VECTOR_COMPRESS, CheckVT) ||
3027 TLI.isOperationCustom(ISD::VECTOR_COMPRESS, CheckVT)) {
3028 HasCustomLowering = true;
3029 break;
3030 }
3031 CheckVT = CheckVT.getHalfNumVectorElementsVT(*DAG.getContext());
3032 }
3033
3034 SDValue Passthru = N->getOperand(2);
3035 if (!HasCustomLowering) {
3036 SDValue Compressed = TLI.expandVECTOR_COMPRESS(N, DAG);
3037 std::tie(Lo, Hi) = DAG.SplitVector(Compressed, DL, LoVT, HiVT);
3038 return;
3039 }
3040
3041 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
3042 SDValue Mask = N->getOperand(1);
3043 SDValue LoMask, HiMask;
3044 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
3045 std::tie(LoMask, HiMask) = SplitMask(Mask);
3046
3047 SDValue UndefPassthru = DAG.getPOISON(LoVT);
3048 Lo = DAG.getNode(ISD::VECTOR_COMPRESS, DL, LoVT, Lo, LoMask, UndefPassthru);
3049 Hi = DAG.getNode(ISD::VECTOR_COMPRESS, DL, HiVT, Hi, HiMask, UndefPassthru);
3050
3051 SDValue StackPtr = DAG.CreateStackTemporary(
3052 VecVT.getStoreSize(), DAG.getReducedAlign(VecVT, /*UseABI=*/false));
3053 MachineFunction &MF = DAG.getMachineFunction();
3054 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
3055 MF, cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex());
3056
3057 EVT LoMaskVT = LoMask.getValueType();
3058 assert(LoMaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
3059
3060 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
3061 EVT WideLoMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
3062 LoMaskVT.getVectorElementCount());
3063 SDValue WideLoMask = DAG.getNode(ISD::ZERO_EXTEND, DL, WideLoMaskVT, LoMask);
3064 SDValue Offset = DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, WideLoMask);
3065 Offset = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Offset);
3066
3067 SDValue Chain = DAG.getEntryNode();
3068 Chain = DAG.getStore(Chain, DL, Lo, StackPtr, PtrInfo);
3069 Chain = DAG.getStore(Chain, DL, Hi, Offset,
3071
3072 SDValue Compressed = DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo);
3073 if (!Passthru.isUndef()) {
3074 // Compress the input mask so only inactive lanes of the result are replaced
3075 // by their passthrough value.
3076 EVT MaskVT = Mask.getValueType();
3077 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
3078 MaskVT.getVectorElementCount());
3079 SDValue WideMask = DAG.getNode(ISD::ZERO_EXTEND, DL, WideMaskVT, Mask);
3080 SDValue NumActiveElts =
3081 DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, WideMask);
3082
3083 SDValue StepVector = DAG.getStepVector(DL, WideMaskVT);
3084 SDValue SplatNumActiveElts = DAG.getSplat(WideMaskVT, DL, NumActiveElts);
3085 SDValue CompressedMask =
3086 DAG.getSetCC(DL, MaskVT, StepVector, SplatNumActiveElts, ISD::SETULT);
3087
3088 Compressed = DAG.getNode(ISD::VSELECT, DL, VecVT, CompressedMask,
3089 Compressed, Passthru);
3090 }
3091 std::tie(Lo, Hi) = DAG.SplitVector(Compressed, DL);
3092}
3093
3094void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE_VAR(SDNode *N, SDValue &Lo,
3095 SDValue &Hi) {
3096 // TODO: Split into half-width shuffles when the target lowers them natively.
3097 SDValue Res = TLI.expandVECTOR_SHUFFLE_VAR(N, DAG);
3098 std::tie(Lo, Hi) = DAG.SplitVector(Res, SDLoc(N));
3099}
3100
3101void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
3102 assert(N->getValueType(0).isVector() &&
3103 N->getOperand(0).getValueType().isVector() &&
3104 "Operand types must be vectors");
3105
3106 EVT LoVT, HiVT;
3107 SDLoc DL(N);
3108 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
3109
3110 // If the input also splits, handle it directly. Otherwise split it by hand.
3111 SDValue LL, LH, RL, RH;
3112 if (getTypeAction(N->getOperand(0).getValueType()) ==
3114 GetSplitVector(N->getOperand(0), LL, LH);
3115 else
3116 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
3117
3118 if (getTypeAction(N->getOperand(1).getValueType()) ==
3120 GetSplitVector(N->getOperand(1), RL, RH);
3121 else
3122 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
3123
3124 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
3125 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
3126}
3127
3128void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
3129 SDValue &Hi) {
3130 // Get the dest types - they may not match the input types, e.g. int_to_fp.
3131 EVT LoVT, HiVT;
3132 SDLoc dl(N);
3133 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
3134
3135 // If the input also splits, handle it directly for a compile time speedup.
3136 // Otherwise split it by hand.
3137 EVT InVT = N->getOperand(0).getValueType();
3138 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
3139 GetSplitVector(N->getOperand(0), Lo, Hi);
3140 else
3141 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
3142
3143 const SDNodeFlags Flags = N->getFlags();
3144 unsigned Opcode = N->getOpcode();
3145 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
3146 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), N->getOperand(2),
3147 N->getOperand(3), Flags);
3148 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), N->getOperand(2),
3149 N->getOperand(3), Flags);
3150 return;
3151 }
3152
3153 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
3155 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
3156 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
3157 } else {
3158 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
3159 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
3160 }
3161}
3162
3163void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
3164 SDValue &Hi) {
3165 SDLoc dl(N);
3166 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
3167
3168 // If the input also splits, handle it directly for a compile time speedup.
3169 // Otherwise split it by hand.
3170 EVT InVT = N->getOperand(0).getValueType();
3171 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
3172 GetSplitVector(N->getOperand(0), Lo, Hi);
3173 else
3174 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
3175
3176 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
3177 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3178 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3179 SDNodeFlags Flags = AddrSpaceCastN->getFlags();
3180 Lo = DAG.getAddrSpaceCast(dl, LoVT, Lo, SrcAS, DestAS, Flags);
3181 Hi = DAG.getAddrSpaceCast(dl, HiVT, Hi, SrcAS, DestAS, Flags);
3182}
3183
3184void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
3185 unsigned ResNo,
3186 SDValue &Lo,
3187 SDValue &Hi) {
3188 SDLoc dl(N);
3189 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
3190 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(N->getValueType(1));
3191
3192 // If the input also splits, handle it directly for a compile time speedup.
3193 // Otherwise split it by hand.
3194 EVT InVT = N->getOperand(0).getValueType();
3195 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
3196 GetSplitVector(N->getOperand(0), Lo, Hi);
3197 else
3198 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
3199
3200 Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo, N->getFlags());
3201 Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi, N->getFlags());
3202
3203 SDNode *HiNode = Hi.getNode();
3204 SDNode *LoNode = Lo.getNode();
3205
3206 // Replace the other vector result not being explicitly split here.
3207 unsigned OtherNo = 1 - ResNo;
3208 EVT OtherVT = N->getValueType(OtherNo);
3209 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
3210 SetSplitVector(SDValue(N, OtherNo), SDValue(LoNode, OtherNo),
3211 SDValue(HiNode, OtherNo));
3212 } else {
3213 SDValue OtherVal =
3214 DAG.getNode(ISD::CONCAT_VECTORS, dl, OtherVT, SDValue(LoNode, OtherNo),
3215 SDValue(HiNode, OtherNo));
3216 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3217 }
3218}
3219
3220void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3221 SDValue &Hi) {
3222 SDLoc dl(N);
3223 EVT SrcVT = N->getOperand(0).getValueType();
3224 EVT DestVT = N->getValueType(0);
3225 EVT LoVT, HiVT;
3226 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
3227
3228 // We can do better than a generic split operation if the extend is doing
3229 // more than just doubling the width of the elements and the following are
3230 // true:
3231 // - The number of vector elements is even,
3232 // - the source type is legal,
3233 // - the type of a split source is illegal,
3234 // - the type of an extended (by doubling element size) source is legal, and
3235 // - the type of that extended source when split is legal.
3236 //
3237 // This won't necessarily completely legalize the operation, but it will
3238 // more effectively move in the right direction and prevent falling down
3239 // to scalarization in many cases due to the input vector being split too
3240 // far.
3241 if (SrcVT.getVectorElementCount().isKnownEven() &&
3242 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3243 LLVMContext &Ctx = *DAG.getContext();
3244 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
3245 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
3246
3247 EVT SplitLoVT, SplitHiVT;
3248 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
3249 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
3250 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
3251 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3252 N->dump(&DAG); dbgs() << "\n");
3253 // Extend the source vector by one step.
3254 SDValue NewSrc =
3255 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
3256 // Get the low and high halves of the new, extended one step, vector.
3257 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
3258 // Extend those vector halves the rest of the way.
3259 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
3260 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
3261 return;
3262 }
3263 }
3264 // Fall back to the generic unary operator splitting otherwise.
3265 SplitVecRes_UnaryOp(N, Lo, Hi);
3266}
3267
3268void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3269 SDValue &Lo, SDValue &Hi) {
3270 // The low and high parts of the original input give four input vectors.
3271 SDValue Inputs[4];
3272 SDLoc DL(N);
3273 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
3274 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
3275 EVT NewVT = Inputs[0].getValueType();
3276 unsigned NewElts = NewVT.getVectorNumElements();
3277
3278 auto &&IsConstant = [](const SDValue &N) {
3279 APInt SplatValue;
3280 return N.getResNo() == 0 && (ISD::isConstantSplatVector(N, SplatValue) ||
3282 };
3283 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3284 SDValue &Input2,
3285 ArrayRef<int> Mask) {
3286 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3287 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3288 "Expected build vector node.");
3289 EVT EltVT = NewVT.getVectorElementType();
3290 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(EltVT));
3291 for (unsigned I = 0; I < NewElts; ++I) {
3292 if (Mask[I] == PoisonMaskElem)
3293 continue;
3294 unsigned Idx = Mask[I];
3295 if (Idx >= NewElts)
3296 Ops[I] = Input2.getOperand(Idx - NewElts);
3297 else
3298 Ops[I] = Input1.getOperand(Idx);
3299 // Make the type of all elements the same as the element type.
3300 if (Ops[I].getValueType().bitsGT(EltVT))
3301 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
3302 }
3303 return DAG.getBuildVector(NewVT, DL, Ops);
3304 };
3305
3306 // If Lo or Hi uses elements from at most two of the four input vectors, then
3307 // express it as a vector shuffle of those two inputs. Otherwise extract the
3308 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3309 SmallVector<int> OrigMask(N->getMask());
3310 // Try to pack incoming shuffles/inputs.
3311 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3312 &DL](SmallVectorImpl<int> &Mask) {
3313 // Check if all inputs are shuffles of the same operands or non-shuffles.
3314 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3315 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3316 SDValue Input = Inputs[Idx];
3317 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
3318 if (!Shuffle ||
3319 Input.getOperand(0).getValueType() != Input.getValueType())
3320 continue;
3321 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
3322 .push_back(Idx);
3323 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
3324 .push_back(Idx);
3325 }
3326 for (auto &P : ShufflesIdxs) {
3327 if (P.second.size() < 2)
3328 continue;
3329 // Use shuffles operands instead of shuffles themselves.
3330 // 1. Adjust mask.
3331 for (int &Idx : Mask) {
3332 if (Idx == PoisonMaskElem)
3333 continue;
3334 unsigned SrcRegIdx = Idx / NewElts;
3335 if (Inputs[SrcRegIdx].isUndef()) {
3337 continue;
3338 }
3339 auto *Shuffle =
3340 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
3341 if (!Shuffle || !is_contained(P.second, SrcRegIdx))
3342 continue;
3343 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3344 if (MaskElt == PoisonMaskElem) {
3346 continue;
3347 }
3348 Idx = MaskElt % NewElts +
3349 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
3350 ? 0
3351 : 1] *
3352 NewElts;
3353 }
3354 // 2. Update inputs.
3355 Inputs[P.second[0]] = P.first.first;
3356 Inputs[P.second[1]] = P.first.second;
3357 // Clear the pair data.
3358 P.second.clear();
3359 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
3360 }
3361 // Check if any concat_vectors can be simplified.
3362 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3363 for (int &Idx : Mask) {
3364 if (Idx == PoisonMaskElem)
3365 continue;
3366 unsigned SrcRegIdx = Idx / NewElts;
3367 if (Inputs[SrcRegIdx].isUndef()) {
3369 continue;
3370 }
3372 getTypeAction(Inputs[SrcRegIdx].getValueType());
3373 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3374 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3375 !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
3376 (TypeAction == TargetLowering::TypeLegal ||
3377 TypeAction == TargetLowering::TypeWidenVector))
3378 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3379 }
3380 if (UsedSubVector.count() > 1) {
3382 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3383 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
3384 continue;
3385 if (Pairs.empty() || Pairs.back().size() == 2)
3386 Pairs.emplace_back();
3387 if (UsedSubVector.test(2 * I)) {
3388 Pairs.back().emplace_back(I, 0);
3389 } else {
3390 assert(UsedSubVector.test(2 * I + 1) &&
3391 "Expected to be used one of the subvectors.");
3392 Pairs.back().emplace_back(I, 1);
3393 }
3394 }
3395 if (!Pairs.empty() && Pairs.front().size() > 1) {
3396 // Adjust mask.
3397 for (int &Idx : Mask) {
3398 if (Idx == PoisonMaskElem)
3399 continue;
3400 unsigned SrcRegIdx = Idx / NewElts;
3401 auto *It = find_if(
3402 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3403 return Idxs.front().first == SrcRegIdx ||
3404 Idxs.back().first == SrcRegIdx;
3405 });
3406 if (It == Pairs.end())
3407 continue;
3408 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3409 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3410 }
3411 // Adjust inputs.
3412 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3413 Inputs[Idxs.front().first] = DAG.getNode(
3415 Inputs[Idxs.front().first].getValueType(),
3416 Inputs[Idxs.front().first].getOperand(Idxs.front().second),
3417 Inputs[Idxs.back().first].getOperand(Idxs.back().second));
3418 }
3419 }
3420 }
3421 bool Changed;
3422 do {
3423 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3424 // reused operands.
3425 Changed = false;
3426 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3427 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
3428 if (!Shuffle)
3429 continue;
3430 if (Shuffle->getOperand(0).getValueType() != NewVT)
3431 continue;
3432 int Op = -1;
3433 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
3434 !Shuffle->isSplat()) {
3435 Op = 0;
3436 } else if (!Inputs[I].hasOneUse() &&
3437 !Shuffle->getOperand(1).isUndef()) {
3438 // Find the only used operand, if possible.
3439 for (int &Idx : Mask) {
3440 if (Idx == PoisonMaskElem)
3441 continue;
3442 unsigned SrcRegIdx = Idx / NewElts;
3443 if (SrcRegIdx != I)
3444 continue;
3445 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3446 if (MaskElt == PoisonMaskElem) {
3448 continue;
3449 }
3450 int OpIdx = MaskElt / NewElts;
3451 if (Op == -1) {
3452 Op = OpIdx;
3453 continue;
3454 }
3455 if (Op != OpIdx) {
3456 Op = -1;
3457 break;
3458 }
3459 }
3460 }
3461 if (Op < 0) {
3462 // Try to check if one of the shuffle operands is used already.
3463 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3464 if (Shuffle->getOperand(OpIdx).isUndef())
3465 continue;
3466 auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
3467 if (It == std::end(Inputs))
3468 continue;
3469 int FoundOp = std::distance(std::begin(Inputs), It);
3470 // Found that operand is used already.
3471 // 1. Fix the mask for the reused operand.
3472 for (int &Idx : Mask) {
3473 if (Idx == PoisonMaskElem)
3474 continue;
3475 unsigned SrcRegIdx = Idx / NewElts;
3476 if (SrcRegIdx != I)
3477 continue;
3478 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3479 if (MaskElt == PoisonMaskElem) {
3481 continue;
3482 }
3483 int MaskIdx = MaskElt / NewElts;
3484 if (OpIdx == MaskIdx)
3485 Idx = MaskElt % NewElts + FoundOp * NewElts;
3486 }
3487 // 2. Set Op to the unused OpIdx.
3488 Op = (OpIdx + 1) % 2;
3489 break;
3490 }
3491 }
3492 if (Op >= 0) {
3493 Changed = true;
3494 Inputs[I] = Shuffle->getOperand(Op);
3495 // Adjust mask.
3496 for (int &Idx : Mask) {
3497 if (Idx == PoisonMaskElem)
3498 continue;
3499 unsigned SrcRegIdx = Idx / NewElts;
3500 if (SrcRegIdx != I)
3501 continue;
3502 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3503 int OpIdx = MaskElt / NewElts;
3504 if (OpIdx != Op)
3505 continue;
3506 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3507 }
3508 }
3509 }
3510 } while (Changed);
3511 };
3512 TryPeekThroughShufflesInputs(OrigMask);
3513 // Proces unique inputs.
3514 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3515 NewElts](SmallVectorImpl<int> &Mask) {
3516 SetVector<SDValue> UniqueInputs;
3517 SetVector<SDValue> UniqueConstantInputs;
3518 for (const auto &I : Inputs) {
3519 if (IsConstant(I))
3520 UniqueConstantInputs.insert(I);
3521 else if (!I.isUndef())
3522 UniqueInputs.insert(I);
3523 }
3524 // Adjust mask in case of reused inputs. Also, need to insert constant
3525 // inputs at first, otherwise it affects the final outcome.
3526 if (UniqueInputs.size() != std::size(Inputs)) {
3527 auto &&UniqueVec = UniqueInputs.takeVector();
3528 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3529 unsigned ConstNum = UniqueConstantVec.size();
3530 for (int &Idx : Mask) {
3531 if (Idx == PoisonMaskElem)
3532 continue;
3533 unsigned SrcRegIdx = Idx / NewElts;
3534 if (Inputs[SrcRegIdx].isUndef()) {
3536 continue;
3537 }
3538 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
3539 if (It != UniqueConstantVec.end()) {
3540 Idx = (Idx % NewElts) +
3541 NewElts * std::distance(UniqueConstantVec.begin(), It);
3542 assert(Idx >= 0 && "Expected defined mask idx.");
3543 continue;
3544 }
3545 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
3546 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3547 Idx = (Idx % NewElts) +
3548 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
3549 assert(Idx >= 0 && "Expected defined mask idx.");
3550 }
3551 copy(UniqueConstantVec, std::begin(Inputs));
3552 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
3553 }
3554 };
3555 MakeUniqueInputs(OrigMask);
3556 SDValue OrigInputs[4];
3557 copy(Inputs, std::begin(OrigInputs));
3558 for (unsigned High = 0; High < 2; ++High) {
3559 SDValue &Output = High ? Hi : Lo;
3560
3561 // Build a shuffle mask for the output, discovering on the fly which
3562 // input vectors to use as shuffle operands.
3563 unsigned FirstMaskIdx = High * NewElts;
3564 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3565 copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
3566 assert(!Output && "Expected default initialized initial value.");
3567 TryPeekThroughShufflesInputs(Mask);
3568 MakeUniqueInputs(Mask);
3569 SDValue TmpInputs[4];
3570 copy(Inputs, std::begin(TmpInputs));
3571 // Track changes in the output registers.
3572 int UsedIdx = -1;
3573 bool SecondIteration = false;
3574 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3575 if (UsedIdx < 0) {
3576 UsedIdx = Idx;
3577 return false;
3578 }
3579 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3580 SecondIteration = true;
3581 return SecondIteration;
3582 };
3584 Mask, std::size(Inputs), std::size(Inputs),
3585 /*NumOfUsedRegs=*/1,
3586 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(NewVT); },
3587 [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3588 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3589 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3590 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3591 else
3592 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
3593 DAG.getPOISON(NewVT), Mask);
3594 Inputs[Idx] = Output;
3595 },
3596 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3597 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3598 unsigned Idx2, bool /*Unused*/) {
3599 if (AccumulateResults(Idx1)) {
3600 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3601 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3602 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3603 else
3604 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
3605 Inputs[Idx2], Mask);
3606 } else {
3607 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3608 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3609 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3610 else
3611 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
3612 TmpInputs[Idx2], Mask);
3613 }
3614 Inputs[Idx1] = Output;
3615 });
3616 copy(OrigInputs, std::begin(Inputs));
3617 }
3618}
3619
3620void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3621 EVT OVT = N->getValueType(0);
3622 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
3623 SDValue Chain = N->getOperand(0);
3624 SDValue Ptr = N->getOperand(1);
3625 SDValue SV = N->getOperand(2);
3626 SDLoc dl(N);
3627
3628 const Align Alignment =
3629 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
3630
3631 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
3632 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
3633 Chain = Hi.getValue(1);
3634
3635 // Modified the chain - switch anything that used the old chain to use
3636 // the new one.
3637 ReplaceValueWith(SDValue(N, 1), Chain);
3638}
3639
3640void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3641 SDValue &Hi) {
3642 EVT DstVTLo, DstVTHi;
3643 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
3644 SDLoc dl(N);
3645
3646 SDValue SrcLo, SrcHi;
3647 EVT SrcVT = N->getOperand(0).getValueType();
3648 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
3649 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
3650 else
3651 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
3652
3653 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
3654 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
3655}
3656
3657void DAGTypeLegalizer::SplitVecRes_VECTOR_REPEAT(SDNode *N, SDValue &Lo,
3658 SDValue &Hi) {
3659 EVT VT = N->getValueType(0);
3660 SDValue Src = N->getOperand(0);
3661 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
3662 assert(LoVT == HiVT && "Expected equal split types");
3663
3664 // Use smaller even/odd source vectors so their broadcasts can be
3665 // reinterleaved in the original lane order for every value of vscale.
3666 SDLoc DL(N);
3667 auto [SrcLo, SrcHi] = DAG.SplitVector(Src, DL);
3668 EVT SplitSrcVT = SrcLo.getValueType();
3669 SDValue Deinterleaved =
3670 DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
3671 DAG.getVTList(SplitSrcVT, SplitSrcVT), SrcLo, SrcHi);
3672 SDValue Even =
3673 DAG.getNode(ISD::VECTOR_REPEAT, DL, LoVT, Deinterleaved.getValue(0));
3674 SDValue Odd =
3675 DAG.getNode(ISD::VECTOR_REPEAT, DL, LoVT, Deinterleaved.getValue(1));
3676 SDValue Interleaved = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
3677 DAG.getVTList(LoVT, LoVT), Even, Odd);
3678 Lo = Interleaved.getValue(0);
3679 Hi = Interleaved.getValue(1);
3680}
3681
3682void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3683 SDValue &Hi) {
3684 SDValue InLo, InHi;
3685 GetSplitVector(N->getOperand(0), InLo, InHi);
3686 SDLoc DL(N);
3687
3688 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
3689 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
3690}
3691
3692void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3693 SDValue &Hi) {
3694 SDLoc DL(N);
3695
3696 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
3697 std::tie(Lo, Hi) = DAG.SplitVector(Expanded, DL);
3698}
3699
3700void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3701 SDValue &Hi) {
3702 EVT VT = N->getValueType(0);
3703 SDValue Val = N->getOperand(0);
3704 SDValue Mask = N->getOperand(1);
3705 SDValue EVL = N->getOperand(2);
3706 SDLoc DL(N);
3707
3708 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3709 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3710 // then truncate back.
3711 EVT OrigVT = VT;
3712 if (!VT.getVectorElementType().isByteSized()) {
3713 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3714 WideEltVT = WideEltVT.getRoundIntegerType(*DAG.getContext());
3715 VT = VT.changeVectorElementType(*DAG.getContext(), WideEltVT);
3716 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Val);
3717 }
3718
3719 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3720 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3721
3722 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
3724 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
3725 EVT PtrVT = StackPtr.getValueType();
3726 auto &MF = DAG.getMachineFunction();
3727 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3728 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3729
3730 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3732 Alignment);
3733 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3735 Alignment);
3736
3737 EVL = DAG.getNode(
3738 ISD::UMIN, DL, EVL.getValueType(), DAG.getFreeze(EVL),
3739 DAG.getElementCount(DL, EVL.getValueType(), VT.getVectorElementCount()));
3740
3741 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3742 SDValue NumElemMinus1 =
3743 DAG.getNode(ISD::SUB, DL, PtrVT, DAG.getZExtOrTrunc(EVL, DL, PtrVT),
3744 DAG.getConstant(1, DL, PtrVT));
3745 SDValue StartOffset = DAG.getNode(ISD::MUL, DL, PtrVT, NumElemMinus1,
3746 DAG.getConstant(EltWidth, DL, PtrVT));
3747 SDValue StorePtr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, StartOffset);
3748 SDValue Stride = DAG.getConstant(-(int64_t)EltWidth, DL, PtrVT);
3749
3750 SDValue TrueMask = DAG.getBoolConstant(true, DL, Mask.getValueType(), VT);
3751 SDValue Store = DAG.getStridedStoreVP(DAG.getEntryNode(), DL, Val, StorePtr,
3752 DAG.getPOISON(PtrVT), Stride, TrueMask,
3753 EVL, MemVT, StoreMMO, ISD::UNINDEXED);
3754
3755 SDValue Load = DAG.getLoadVP(VT, DL, Store, StackPtr, Mask, EVL, LoadMMO);
3756
3757 // Truncate back if we widened above.
3758 if (OrigVT != VT)
3759 Load = DAG.getNode(ISD::TRUNCATE, DL, OrigVT, Load);
3760
3761 std::tie(Lo, Hi) = DAG.SplitVector(Load, DL);
3762}
3763
3764void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3765 SDValue &Hi) {
3766 EVT VT = N->getValueType(0);
3767 SDValue V1 = N->getOperand(0);
3768 SDValue V2 = N->getOperand(1);
3769 int64_t Imm = cast<ConstantSDNode>(N->getOperand(2))->getSExtValue();
3770 SDValue Mask = N->getOperand(3);
3771 SDValue EVL1 = N->getOperand(4);
3772 SDValue EVL2 = N->getOperand(5);
3773 SDLoc DL(N);
3774
3775 // Since EVL2 is considered the real VL it gets promoted during
3776 // SelectionDAGBuilder. Promote EVL1 here if needed.
3777 if (getTypeAction(EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3778 EVL1 = ZExtPromotedInteger(EVL1);
3779
3780 // The stack splice addresses elements by byte offset/stride, which breaks for
3781 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3782 // stride is 0. Widen to a byte integer, splice, then truncate back.
3783 EVT OrigVT = VT;
3784 if (!VT.getVectorElementType().isByteSized()) {
3785 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3786 WideEltVT = WideEltVT.getRoundIntegerType(*DAG.getContext());
3787 VT = VT.changeVectorElementType(*DAG.getContext(), WideEltVT);
3788 V1 = DAG.getNode(ISD::ANY_EXTEND, DL, VT, V1);
3789 V2 = DAG.getNode(ISD::ANY_EXTEND, DL, VT, V2);
3790 }
3791
3792 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3793
3794 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
3795 VT.getVectorElementCount() * 2);
3796 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
3797 EVT PtrVT = StackPtr.getValueType();
3798 auto &MF = DAG.getMachineFunction();
3799 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3800 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3801
3802 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3804 Alignment);
3805 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3807 Alignment);
3808
3809 SDValue EltByteSize =
3810 DAG.getTypeSize(DL, PtrVT, VT.getVectorElementType().getStoreSize());
3811 SDValue EVL1Ptr = DAG.getZExtOrTrunc(DAG.getFreeze(EVL1), DL, PtrVT);
3812 SDValue EVL1Bytes = DAG.getNode(ISD::MUL, DL, PtrVT, EVL1Ptr, EltByteSize);
3813 // Clip EVL1Bytes to make sure we stay within the stack object.
3814 SDValue VTBytes = DAG.getTypeSize(DL, PtrVT, VT.getStoreSize());
3815 EVL1Bytes = DAG.getNode(ISD::UMIN, DL, PtrVT, EVL1Bytes, VTBytes);
3816 SDValue StackPtr2 = DAG.getMemBasePlusOffset(StackPtr, EVL1Bytes, DL);
3817 SDValue PoisonPtr = DAG.getPOISON(PtrVT);
3818
3819 SDValue TrueMask = DAG.getBoolConstant(true, DL, Mask.getValueType(), VT);
3820 SDValue StoreV1 =
3821 DAG.getStoreVP(DAG.getEntryNode(), DL, V1, StackPtr, PoisonPtr, TrueMask,
3822 EVL1, V1.getValueType(), StoreMMO, ISD::UNINDEXED);
3823
3825 DAG.getStoreVP(StoreV1, DL, V2, StackPtr2, PoisonPtr, TrueMask, EVL2,
3826 V2.getValueType(), StoreMMO, ISD::UNINDEXED);
3827
3828 SDValue Load;
3829 if (Imm >= 0) {
3830 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VT, N->getOperand(2));
3831 Load = DAG.getLoadVP(VT, DL, StoreV2, StackPtr, Mask, EVL2, LoadMMO);
3832 } else {
3833 uint64_t TrailingElts = -Imm;
3834 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3835 SDValue TrailingBytes = DAG.getConstant(TrailingElts * EltWidth, DL, PtrVT);
3836
3837 // Make sure TrailingBytes doesn't exceed the size of vec1.
3838 SDValue OffsetToV2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, StackPtr);
3839 TrailingBytes =
3840 DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, OffsetToV2);
3841
3842 // Calculate the start address of the spliced result.
3843 StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);
3844 Load = DAG.getLoadVP(VT, DL, StoreV2, StackPtr2, Mask, EVL2, LoadMMO);
3845 }
3846
3847 // Truncate back if we widened above.
3848 if (OrigVT != VT)
3849 Load = DAG.getNode(ISD::TRUNCATE, DL, OrigVT, Load);
3850
3851 EVT LoVT, HiVT;
3852 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(OrigVT);
3853 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Load,
3854 DAG.getVectorIdxConstant(0, DL));
3855 Hi =
3856 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Load,
3857 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
3858}
3859
3860void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3861 SDValue &Hi) {
3862 SDLoc DL(N);
3863 SDValue Acc = N->getOperand(0);
3864 SDValue Input1 = N->getOperand(1);
3865 SDValue Input2 = N->getOperand(2);
3866
3867 SDValue AccLo, AccHi;
3868 GetSplitVector(Acc, AccLo, AccHi);
3869 unsigned Opcode = N->getOpcode();
3870
3871 // If the input types don't need splitting, just accumulate into the
3872 // low part of the accumulator.
3873 if (getTypeAction(Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3874 Lo = DAG.getNode(Opcode, DL, AccLo.getValueType(), AccLo, Input1, Input2);
3875 Hi = AccHi;
3876 return;
3877 }
3878
3879 SDValue Input1Lo, Input1Hi;
3880 SDValue Input2Lo, Input2Hi;
3881 GetSplitVector(Input1, Input1Lo, Input1Hi);
3882 GetSplitVector(Input2, Input2Lo, Input2Hi);
3883 EVT ResultVT = AccLo.getValueType();
3884
3885 Lo = DAG.getNode(Opcode, DL, ResultVT, AccLo, Input1Lo, Input2Lo);
3886 Hi = DAG.getNode(Opcode, DL, ResultVT, AccHi, Input1Hi, Input2Hi);
3887}
3888
3889void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3890 SDValue &Hi) {
3891 SDLoc DL(N);
3892 SDValue Op0 = N->getOperand(0);
3893 SDValue Op1 = N->getOperand(1);
3894 EVT OpVT = Op0.getValueType();
3895
3896 EVT LoVT, HiVT;
3897 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
3898
3899 Lo = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, LoVT, Op0, Op1);
3900 SDValue LoElts = DAG.getElementCount(DL, OpVT, LoVT.getVectorElementCount());
3901 SDValue HiStartVal = DAG.getNode(ISD::UADDSAT, DL, OpVT, Op0, LoElts);
3902 Hi = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, HiVT, HiStartVal, Op1);
3903}
3904
3905void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
3906 SDValue &Hi) {
3907 SDValue SourceLo, SourceHi;
3908 GetSplitVector(N->getOperand(0), SourceLo, SourceHi);
3909 SDValue MaskLo, MaskHi;
3910 GetSplitVector(N->getOperand(2), MaskLo, MaskHi);
3911 SDLoc DL(N);
3912
3913 Lo = DAG.getNode(ISD::VECTOR_MATCH, DL, MaskLo.getValueType(), SourceLo,
3914 N->getOperand(1), MaskLo, N->getFlags());
3915 Hi = DAG.getNode(ISD::VECTOR_MATCH, DL, MaskHi.getValueType(), SourceHi,
3916 N->getOperand(1), MaskHi, N->getFlags());
3917}
3918
3919void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3920 unsigned Factor = N->getNumOperands();
3921
3922 SmallVector<SDValue, 8> Ops(Factor * 2);
3923 for (unsigned i = 0; i != Factor; ++i) {
3924 SDValue OpLo, OpHi;
3925 GetSplitVector(N->getOperand(i), OpLo, OpHi);
3926 Ops[i * 2] = OpLo;
3927 Ops[i * 2 + 1] = OpHi;
3928 }
3929
3930 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3931
3932 SDLoc DL(N);
3933 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
3934 ArrayRef(Ops).slice(0, Factor));
3935 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
3936 ArrayRef(Ops).slice(Factor, Factor));
3937
3938 for (unsigned i = 0; i != Factor; ++i)
3939 SetSplitVector(SDValue(N, i), ResLo.getValue(i), ResHi.getValue(i));
3940}
3941
3942void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3943 unsigned Factor = N->getNumOperands();
3944
3945 SmallVector<SDValue, 8> Ops(Factor * 2);
3946 for (unsigned i = 0; i != Factor; ++i) {
3947 SDValue OpLo, OpHi;
3948 GetSplitVector(N->getOperand(i), OpLo, OpHi);
3949 Ops[i] = OpLo;
3950 Ops[i + Factor] = OpHi;
3951 }
3952
3953 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3954
3955 SDLoc DL(N);
3956 SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
3957 ArrayRef(Ops).slice(0, Factor)),
3958 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
3959 ArrayRef(Ops).slice(Factor, Factor))};
3960
3961 for (unsigned i = 0; i != Factor; ++i) {
3962 unsigned IdxLo = 2 * i;
3963 unsigned IdxHi = 2 * i + 1;
3964 SetSplitVector(SDValue(N, i), Res[IdxLo / Factor].getValue(IdxLo % Factor),
3965 Res[IdxHi / Factor].getValue(IdxHi % Factor));
3966 }
3967}
3968
3969//===----------------------------------------------------------------------===//
3970// Operand Vector Splitting
3971//===----------------------------------------------------------------------===//
3972
3973/// This method is called when the specified operand of the specified node is
3974/// found to need vector splitting. At this point, all of the result types of
3975/// the node are known to be legal, but other operands of the node may need
3976/// legalization as well as the specified one.
3977bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3978 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3979 SDValue Res = SDValue();
3980
3981 // See if the target wants to custom split this node.
3982 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3983 return false;
3984
3985 switch (N->getOpcode()) {
3986 default:
3987#ifndef NDEBUG
3988 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3989 N->dump(&DAG);
3990 dbgs() << "\n";
3991#endif
3992 report_fatal_error("Do not know how to split this operator's "
3993 "operand!\n");
3994
3995 case ISD::STRICT_FSETCC:
3997 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3998 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3999 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
4000 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
4001 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
4002 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
4004 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
4005 break;
4006 case ISD::TRUNCATE:
4007 Res = SplitVecOp_TruncateHelper(N);
4008 break;
4010 case ISD::FP_ROUND:
4013 Res = SplitVecOp_FP_ROUND(N);
4014 break;
4015 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
4016 case ISD::STORE:
4017 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
4018 break;
4019 case ISD::ATOMIC_STORE:
4020 Res = SplitVecOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
4021 break;
4022 case ISD::VP_STORE:
4023 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
4024 break;
4025 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
4026 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo);
4027 break;
4028 case ISD::MSTORE:
4029 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
4030 break;
4031 case ISD::MSCATTER:
4032 case ISD::VP_SCATTER:
4033 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
4034 break;
4035 case ISD::MGATHER:
4036 case ISD::VP_GATHER:
4037 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
4038 break;
4039 case ISD::VSELECT:
4040 Res = SplitVecOp_VSELECT(N, OpNo);
4041 break;
4042 case ISD::MASKED_UDIV:
4043 case ISD::MASKED_SDIV:
4044 case ISD::MASKED_UREM:
4045 case ISD::MASKED_SREM:
4046 Res = SplitVecOp_MaskedBinOp(N, OpNo);
4047 break;
4049 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
4050 break;
4052 Res = SplitVecOp_VECTOR_SHUFFLE_VAR(N, OpNo);
4053 break;
4056 case ISD::SINT_TO_FP:
4057 case ISD::UINT_TO_FP:
4058 if (N->getValueType(0).bitsLT(
4059 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
4060 Res = SplitVecOp_TruncateHelper(N);
4061 else
4062 Res = SplitVecOp_UnaryOp(N);
4063 break;
4066 Res = SplitVecOp_FP_TO_XINT_SAT(N);
4067 break;
4068 case ISD::FP_TO_SINT:
4069 case ISD::FP_TO_UINT:
4073 case ISD::FP_EXTEND:
4074 case ISD::SIGN_EXTEND:
4075 case ISD::ZERO_EXTEND:
4076 case ISD::ANY_EXTEND:
4077 case ISD::FTRUNC:
4078 case ISD::LROUND:
4079 case ISD::LLROUND:
4080 case ISD::LRINT:
4081 case ISD::LLRINT:
4082 Res = SplitVecOp_UnaryOp(N);
4083 break;
4084 case ISD::FLDEXP:
4085 Res = SplitVecOp_FPOpDifferentTypes(N);
4086 break;
4087
4088 case ISD::SCMP:
4089 case ISD::UCMP:
4090 Res = SplitVecOp_CMP(N);
4091 break;
4092
4093 case ISD::FAKE_USE:
4094 Res = SplitVecOp_FAKE_USE(N);
4095 break;
4099 Res = SplitVecOp_ExtVecInRegOp(N);
4100 break;
4101
4104 case ISD::VECREDUCE_ADD:
4105 case ISD::VECREDUCE_MUL:
4106 case ISD::VECREDUCE_AND:
4107 case ISD::VECREDUCE_OR:
4108 case ISD::VECREDUCE_XOR:
4119 Res = SplitVecOp_VECREDUCE(N, OpNo);
4120 break;
4123 Res = SplitVecOp_VECREDUCE_SEQ(N);
4124 break;
4125 case ISD::VP_REDUCE_FADD:
4126 case ISD::VP_REDUCE_SEQ_FADD:
4127 case ISD::VP_REDUCE_FMUL:
4128 case ISD::VP_REDUCE_SEQ_FMUL:
4129 case ISD::VP_REDUCE_ADD:
4130 case ISD::VP_REDUCE_MUL:
4131 case ISD::VP_REDUCE_AND:
4132 case ISD::VP_REDUCE_OR:
4133 case ISD::VP_REDUCE_XOR:
4134 case ISD::VP_REDUCE_SMAX:
4135 case ISD::VP_REDUCE_SMIN:
4136 case ISD::VP_REDUCE_UMAX:
4137 case ISD::VP_REDUCE_UMIN:
4138 case ISD::VP_REDUCE_FMAX:
4139 case ISD::VP_REDUCE_FMIN:
4140 case ISD::VP_REDUCE_FMAXIMUM:
4141 case ISD::VP_REDUCE_FMINIMUM:
4142 Res = SplitVecOp_VP_REDUCE(N, OpNo);
4143 break;
4144 case ISD::CTTZ_ELTS:
4146 Res = SplitVecOp_CttzElts(N);
4147 break;
4148 case ISD::VP_CTTZ_ELTS:
4149 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4150 Res = SplitVecOp_VP_CttzElements(N);
4151 break;
4153 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
4154 break;
4159 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
4160 break;
4161 case ISD::VECTOR_MATCH:
4162 Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
4163 break;
4164 }
4165
4166 // If the result is null, the sub-method took care of registering results etc.
4167 if (!Res.getNode()) return false;
4168
4169 // If the result is N, the sub-method updated N in place. Tell the legalizer
4170 // core about this.
4171 if (Res.getNode() == N)
4172 return true;
4173
4174 if (N->isStrictFPOpcode())
4175 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4176 "Invalid operand expansion");
4177 else
4178 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4179 "Invalid operand expansion");
4180
4181 ReplaceValueWith(SDValue(N, 0), Res);
4182 return false;
4183}
4184
4185SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
4186 SDLoc DL(N);
4187
4188 SDValue LoMask, HiMask;
4189 GetSplitVector(N->getOperand(0), LoMask, HiMask);
4190
4191 EVT VT = N->getValueType(0);
4192 EVT SplitVT = LoMask.getValueType();
4193 ElementCount SplitEC = SplitVT.getVectorElementCount();
4194
4195 // Find the last active in both the low and the high masks.
4196 SDValue LoFind = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, LoMask);
4197 SDValue HiFind = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, HiMask);
4198
4199 // Check if any lane is active in the high mask.
4200 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
4201 // sentinel value for "none active".
4202 SDValue AnyHiActive = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, HiMask);
4203 SDValue Cond = DAG.getBoolExtOrTrunc(AnyHiActive, DL,
4204 getSetCCResultType(MVT::i1), MVT::i1);
4205
4206 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
4207 return DAG.getNode(ISD::SELECT, DL, VT, Cond,
4208 DAG.getNode(ISD::ADD, DL, VT, HiFind,
4209 DAG.getElementCount(DL, VT, SplitEC)),
4210 LoFind);
4211}
4212
4213SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
4214 // The only possibility for an illegal operand is the mask, since result type
4215 // legalization would have handled this node already otherwise.
4216 assert(OpNo == 0 && "Illegal operand must be mask");
4217
4218 SDValue Mask = N->getOperand(0);
4219 SDValue Src0 = N->getOperand(1);
4220 SDValue Src1 = N->getOperand(2);
4221 EVT Src0VT = Src0.getValueType();
4222 SDLoc DL(N);
4223 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4224
4225 SDValue Lo, Hi;
4226 GetSplitVector(N->getOperand(0), Lo, Hi);
4227 assert(Lo.getValueType() == Hi.getValueType() &&
4228 "Lo and Hi have differing types");
4229
4230 EVT LoOpVT, HiOpVT;
4231 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
4232 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4233
4234 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4235 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
4236 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
4237 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
4238
4239 SDValue LoSelect =
4240 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
4241 SDValue HiSelect =
4242 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
4243
4244 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
4245}
4246
4247SDValue DAGTypeLegalizer::SplitVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
4248 assert(OpNo == 2 && "Illegal operand must be mask");
4249
4250 SDLoc DL(N);
4251 auto [LHSLo, LHSHi] = DAG.SplitVector(N->getOperand(0), DL);
4252 auto [RHSLo, RHSHi] = DAG.SplitVector(N->getOperand(1), DL);
4253 SDValue MaskLo, MaskHi;
4254 GetSplitVector(N->getOperand(2), MaskLo, MaskHi);
4255
4256 SDValue Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo,
4257 RHSLo, MaskLo, N->getFlags());
4258 SDValue Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi,
4259 RHSHi, MaskHi, N->getFlags());
4260 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
4261}
4262
4263SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4264 // The only possibility for an illegal operand is the mask, since result type
4265 // legalization would have handled this node already otherwise.
4266 assert(OpNo == 1 && "Illegal operand must be mask");
4267
4268 // To split the mask, we need to split the result type too, so we can just
4269 // reuse that logic here.
4270 SDValue Lo, Hi;
4271 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4272
4273 EVT VecVT = N->getValueType(0);
4274 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VecVT, Lo, Hi);
4275}
4276
4277SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_SHUFFLE_VAR(SDNode *N,
4278 unsigned OpNo) {
4279 // The only possibility for an illegal operand is the mask, since result type
4280 // legalization would have handled this node already otherwise.
4281 assert(OpNo == 1 && "Illegal operand must be mask");
4282 // TODO: Legalize the mask type instead once a target lowers the node.
4283 return TLI.expandVECTOR_SHUFFLE_VAR(N, DAG);
4284}
4285
4286SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4287 EVT ResVT = N->getValueType(0);
4288 SDValue Lo, Hi;
4289 SDLoc dl(N);
4290
4291 SDValue VecOp = N->getOperand(OpNo);
4292 EVT VecVT = VecOp.getValueType();
4293 assert(VecVT.isVector() && "Can only split reduce vector operand");
4294 GetSplitVector(VecOp, Lo, Hi);
4295 EVT LoOpVT, HiOpVT;
4296 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
4297
4298 // Use the appropriate scalar instruction on the split subvectors before
4299 // reducing the now partially reduced smaller vector.
4300 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
4301 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
4302 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
4303}
4304
4305SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4306 EVT ResVT = N->getValueType(0);
4307 SDValue Lo, Hi;
4308 SDLoc dl(N);
4309
4310 SDValue AccOp = N->getOperand(0);
4311 SDValue VecOp = N->getOperand(1);
4312 SDNodeFlags Flags = N->getFlags();
4313
4314 EVT VecVT = VecOp.getValueType();
4315 assert(VecVT.isVector() && "Can only split reduce vector operand");
4316 GetSplitVector(VecOp, Lo, Hi);
4317 EVT LoOpVT, HiOpVT;
4318 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
4319
4320 // Reduce low half.
4321 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
4322
4323 // Reduce high half, using low half result as initial value.
4324 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
4325}
4326
4327SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4328 assert(N->isVPOpcode() && "Expected VP opcode");
4329 assert(OpNo == 1 && "Can only split reduce vector operand");
4330
4331 unsigned Opc = N->getOpcode();
4332 EVT ResVT = N->getValueType(0);
4333 SDValue Lo, Hi;
4334 SDLoc dl(N);
4335
4336 SDValue VecOp = N->getOperand(OpNo);
4337 EVT VecVT = VecOp.getValueType();
4338 assert(VecVT.isVector() && "Can only split reduce vector operand");
4339 GetSplitVector(VecOp, Lo, Hi);
4340
4341 SDValue MaskLo, MaskHi;
4342 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
4343
4344 SDValue EVLLo, EVLHi;
4345 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
4346
4347 const SDNodeFlags Flags = N->getFlags();
4348
4349 SDValue ResLo =
4350 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
4351 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
4352}
4353
4354SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4355 // The result has a legal vector type, but the input needs splitting.
4356 EVT ResVT = N->getValueType(0);
4357 SDValue Lo, Hi;
4358 SDLoc dl(N);
4359 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4360 EVT InVT = Lo.getValueType();
4361
4362 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
4363 InVT.getVectorElementCount());
4364
4365 if (N->isStrictFPOpcode()) {
4366 Lo = DAG.getNode(N->getOpcode(), dl, {OutVT, MVT::Other},
4367 {N->getOperand(0), Lo});
4368 Hi = DAG.getNode(N->getOpcode(), dl, {OutVT, MVT::Other},
4369 {N->getOperand(0), Hi});
4370
4371 // Build a factor node to remember that this operation is independent
4372 // of the other one.
4373 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4374 Hi.getValue(1));
4375
4376 // Legalize the chain result - switch anything that used the old chain to
4377 // use the new one.
4378 ReplaceValueWith(SDValue(N, 1), Ch);
4379 } else {
4380 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
4381 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
4382 }
4383
4384 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
4385}
4386
4387// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4388SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4389 SDValue Lo, Hi;
4390 GetSplitVector(N->getOperand(1), Lo, Hi);
4391 SDValue Chain =
4392 DAG.getNode(ISD::FAKE_USE, SDLoc(), MVT::Other, N->getOperand(0), Lo);
4393 return DAG.getNode(ISD::FAKE_USE, SDLoc(), MVT::Other, Chain, Hi);
4394}
4395
4396SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4397 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4398 // end up being split all the way down to individual components. Convert the
4399 // split pieces into integers and reassemble.
4400 EVT ResVT = N->getValueType(0);
4401 SDValue Lo, Hi;
4402 GetSplitVector(N->getOperand(0), Lo, Hi);
4403 SDLoc dl(N);
4404
4405 if (ResVT.isScalableVector()) {
4406 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(ResVT);
4407 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
4408 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
4409 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
4410 }
4411
4412 Lo = BitConvertToInteger(Lo);
4413 Hi = BitConvertToInteger(Hi);
4414
4415 if (DAG.getDataLayout().isBigEndian())
4416 std::swap(Lo, Hi);
4417
4418 return DAG.getNode(ISD::BITCAST, dl, ResVT, JoinIntegers(Lo, Hi));
4419}
4420
4421SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4422 unsigned OpNo) {
4423 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4424 // We know that the result type is legal.
4425 EVT ResVT = N->getValueType(0);
4426
4427 SDValue Vec = N->getOperand(0);
4428 SDValue SubVec = N->getOperand(1);
4429 SDValue Idx = N->getOperand(2);
4430 SDLoc dl(N);
4431
4432 SDValue Lo, Hi;
4433 GetSplitVector(SubVec, Lo, Hi);
4434
4435 uint64_t IdxVal = Idx->getAsZExtVal();
4437
4438 SDValue FirstInsertion =
4439 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
4440 SDValue SecondInsertion =
4441 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
4442 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
4443
4444 return SecondInsertion;
4445}
4446
4447SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4448 // We know that the extracted result type is legal.
4449 EVT SubVT = N->getValueType(0);
4450 SDValue Idx = N->getOperand(1);
4451 SDLoc dl(N);
4452 SDValue Lo, Hi;
4453
4454 GetSplitVector(N->getOperand(0), Lo, Hi);
4455
4456 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4457 // Note: For scalable vectors, the index is scaled by vscale.
4458 ElementCount IdxVal =
4459 ElementCount::get(Idx->getAsZExtVal(), SubVT.isScalableVector());
4460 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4461
4462 EVT SrcVT = N->getOperand(0).getValueType();
4463 ElementCount NumResultElts = SubVT.getVectorElementCount();
4464
4465 // If the extracted elements are all in the low half, do a simple extract.
4466 if (ElementCount::isKnownLE(IdxVal + NumResultElts, LoElts))
4467 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
4468
4469 unsigned LoEltsMin = LoElts.getKnownMinValue();
4470 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4471 SrcVT.isFixedLengthVector()) {
4472 // Extracted subvector crosses vector split, so we need to blend the two
4473 // halves.
4474 // TODO: May be able to emit partial extract_subvector.
4476 Elts.reserve(NumResultElts.getFixedValue());
4477
4478 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4479 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4480 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4481 DAG.ExtractVectorElements(Lo, Elts, /*Start=*/IdxValMin,
4482 /*Count=*/LoEltsMin - IdxValMin);
4483 DAG.ExtractVectorElements(Hi, Elts, /*Start=*/0,
4484 /*Count=*/SubVT.getVectorNumElements() -
4485 Elts.size());
4486 return DAG.getBuildVector(SubVT, dl, Elts);
4487 }
4488
4489 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4490 ElementCount ExtractIdx = IdxVal - LoElts;
4491 if (ExtractIdx.isKnownMultipleOf(NumResultElts))
4492 return DAG.getExtractSubvector(dl, SubVT, Hi,
4493 ExtractIdx.getKnownMinValue());
4494
4495 EVT HiVT = Hi.getValueType();
4496 assert(HiVT.isFixedLengthVector() &&
4497 "Only fixed-vector extracts are supported in this case");
4498
4499 // We cannot create an extract_subvector that isn't a multiple of the
4500 // result size, which may go out of bounds for the last elements. Shuffle
4501 // the desired elements down to 0 and do a simple 0 extract.
4502 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4503 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4504 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4505
4506 SDValue Shuffle =
4507 DAG.getVectorShuffle(HiVT, dl, Hi, DAG.getPOISON(HiVT), Mask);
4508 return DAG.getExtractSubvector(dl, SubVT, Shuffle, 0);
4509 }
4510
4511 // After this point the DAG node only permits extracting fixed-width
4512 // subvectors from scalable vectors.
4513 assert(SubVT.isFixedLengthVector() &&
4514 "Extracting scalable subvector from fixed-width unsupported");
4515
4516 // If the element type is i1 and we're not promoting the result, then we may
4517 // end up loading the wrong data since the bits are packed tightly into
4518 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4519 // type at index 4, then we will load a byte starting at index 0.
4520 if (SubVT.getScalarType() == MVT::i1)
4521 report_fatal_error("Don't know how to extract fixed-width predicate "
4522 "subvector from a scalable predicate vector");
4523
4524 // Spill the vector to the stack. We should use the alignment for
4525 // the smallest part.
4526 SDValue Vec = N->getOperand(0);
4527 EVT VecVT = Vec.getValueType();
4528 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
4530 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
4531 auto &MF = DAG.getMachineFunction();
4532 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4533 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
4534
4535 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
4536 SmallestAlign);
4537
4538 // Extract the subvector by loading the correct part.
4539 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
4540
4541 return DAG.getLoad(
4542 SubVT, dl, Store, StackPtr,
4543 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
4544}
4545
4546SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4547 SDValue Vec = N->getOperand(0);
4548 SDValue Idx = N->getOperand(1);
4549 EVT VecVT = Vec.getValueType();
4550
4551 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) {
4552 uint64_t IdxVal = Index->getZExtValue();
4553
4554 SDValue Lo, Hi;
4555 GetSplitVector(Vec, Lo, Hi);
4556
4557 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4558
4559 if (IdxVal < LoElts)
4560 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
4561 else if (!Vec.getValueType().isScalableVector())
4562 return SDValue(DAG.UpdateNodeOperands(N, Hi,
4563 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
4564 Idx.getValueType())), 0);
4565 }
4566
4567 // See if the target wants to custom expand this node.
4568 if (CustomLowerNode(N, N->getValueType(0), true))
4569 return SDValue();
4570
4571 // Make the vector elements byte-addressable if they aren't already.
4572 SDLoc dl(N);
4573 EVT EltVT = VecVT.getVectorElementType();
4574 if (!EltVT.isByteSized()) {
4575 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(*DAG.getContext());
4576 VecVT = VecVT.changeElementType(*DAG.getContext(), EltVT);
4577 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
4578 SDValue NewExtract =
4579 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vec, Idx);
4580 return DAG.getAnyExtOrTrunc(NewExtract, dl, N->getValueType(0));
4581 }
4582
4583 // Store the vector to the stack.
4584 // In cases where the vector is illegal it will be broken down into parts
4585 // and stored in parts - we should use the alignment for the smallest part.
4586 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
4588 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
4589 auto &MF = DAG.getMachineFunction();
4590 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4591 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
4592 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
4593 SmallestAlign);
4594
4595 // Load back the required element.
4596 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
4597
4598 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4599 // type, leaving the high bits undefined. But it can't truncate.
4600 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4601
4602 return DAG.getExtLoad(
4603 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
4604 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
4605 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
4606}
4607
4608SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4609 SDValue Lo, Hi;
4610
4611 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4612 // splitting the result has the same effect as splitting the input operand.
4613 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4614
4615 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
4616}
4617
4618SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4619 (void)OpNo;
4620 SDValue Lo, Hi;
4621 SplitVecRes_Gather(N, Lo, Hi);
4622
4623 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
4624 ReplaceValueWith(SDValue(N, 0), Res);
4625 return SDValue();
4626}
4627
4628SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4629 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4630 SDValue Ch = N->getChain();
4631 SDValue Ptr = N->getBasePtr();
4632 SDValue Offset = N->getOffset();
4633 assert(Offset.isUndef() && "Unexpected VP store offset");
4634 SDValue Mask = N->getMask();
4635 SDValue EVL = N->getVectorLength();
4636 SDValue Data = N->getValue();
4637 Align Alignment = N->getBaseAlign();
4638 SDLoc DL(N);
4639
4640 SDValue DataLo, DataHi;
4641 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
4642 // Split Data operand
4643 GetSplitVector(Data, DataLo, DataHi);
4644 else
4645 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
4646
4647 // Split Mask operand
4648 SDValue MaskLo, MaskHi;
4649 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4650 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
4651 } else {
4652 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
4653 GetSplitVector(Mask, MaskLo, MaskHi);
4654 else
4655 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
4656 }
4657
4658 EVT MemoryVT = N->getMemoryVT();
4659 EVT LoMemVT, HiMemVT;
4660 bool HiIsEmpty = false;
4661 std::tie(LoMemVT, HiMemVT) =
4662 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
4663
4664 // Split EVL
4665 SDValue EVLLo, EVLHi;
4666 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
4667
4668 SDValue Lo, Hi;
4669 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4670 N->getPointerInfo(), MachineMemOperand::MOStore,
4672 MMOMetadata(N->getAAInfo(), N->getRanges()));
4673
4674 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
4675 N->getAddressingMode(), N->isTruncatingStore(),
4676 N->isCompressingStore());
4677
4678 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4679 if (HiIsEmpty)
4680 return Lo;
4681
4682 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
4683 N->isCompressingStore());
4684
4685 MachinePointerInfo MPI;
4686 if (LoMemVT.isScalableVector()) {
4687 Alignment = commonAlignment(Alignment,
4688 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4689 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4690 } else
4691 MPI = N->getPointerInfo().getWithOffset(
4692 LoMemVT.getStoreSize().getFixedValue());
4693
4694 MMO = DAG.getMachineFunction().getMachineMemOperand(
4696 Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
4697
4698 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
4699 N->getAddressingMode(), N->isTruncatingStore(),
4700 N->isCompressingStore());
4701
4702 // Build a factor node to remember that this store is independent of the
4703 // other one.
4704 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
4705}
4706
4707SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4708 unsigned OpNo) {
4709 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4710 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4711
4712 SDLoc DL(N);
4713
4714 SDValue Data = N->getValue();
4715 SDValue LoData, HiData;
4716 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
4717 GetSplitVector(Data, LoData, HiData);
4718 else
4719 std::tie(LoData, HiData) = DAG.SplitVector(Data, DL);
4720
4721 EVT LoMemVT, HiMemVT;
4722 bool HiIsEmpty = false;
4723 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
4724 N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty);
4725
4726 SDValue Mask = N->getMask();
4727 SDValue LoMask, HiMask;
4728 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4729 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
4730 else if (getTypeAction(Mask.getValueType()) ==
4732 GetSplitVector(Mask, LoMask, HiMask);
4733 else
4734 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
4735
4736 SDValue LoEVL, HiEVL;
4737 std::tie(LoEVL, HiEVL) =
4738 DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL);
4739
4740 // Generate the low vp_strided_store
4741 SDValue Lo = DAG.getStridedStoreVP(
4742 N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(),
4743 N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(),
4744 N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore());
4745
4746 // If the high vp_strided_store has zero storage size, only the low
4747 // vp_strided_store is needed.
4748 if (HiIsEmpty)
4749 return Lo;
4750
4751 // Generate the high vp_strided_store.
4752 // To calculate the high base address, we need to sum to the low base
4753 // address stride number of bytes for each element already stored by low,
4754 // that is: Ptr = Ptr + (LoEVL * Stride)
4755 EVT PtrVT = N->getBasePtr().getValueType();
4757 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
4758 DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT));
4759 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment);
4760
4761 Align Alignment = N->getBaseAlign();
4762 if (LoMemVT.isScalableVector())
4763 Alignment = commonAlignment(Alignment,
4764 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4765
4766 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4767 MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4769 Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
4770
4771 SDValue Hi = DAG.getStridedStoreVP(
4772 N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
4773 HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(),
4774 N->isCompressingStore());
4775
4776 // Build a factor node to remember that this store is independent of the
4777 // other one.
4778 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
4779}
4780
4781SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4782 unsigned OpNo) {
4783 assert(N->isUnindexed() && "Indexed masked store of vector?");
4784 SDValue Ch = N->getChain();
4785 SDValue Ptr = N->getBasePtr();
4786 SDValue Offset = N->getOffset();
4787 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4788 SDValue Mask = N->getMask();
4789 SDValue Data = N->getValue();
4790 Align Alignment = N->getBaseAlign();
4791 SDLoc DL(N);
4792
4793 SDValue DataLo, DataHi;
4794 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
4795 // Split Data operand
4796 GetSplitVector(Data, DataLo, DataHi);
4797 else
4798 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
4799
4800 // Split Mask operand
4801 SDValue MaskLo, MaskHi;
4802 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4803 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
4804 } else {
4805 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
4806 GetSplitVector(Mask, MaskLo, MaskHi);
4807 else
4808 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
4809 }
4810
4811 EVT MemoryVT = N->getMemoryVT();
4812 EVT LoMemVT, HiMemVT;
4813 bool HiIsEmpty = false;
4814 std::tie(LoMemVT, HiMemVT) =
4815 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
4816
4817 SDValue Lo, Hi, Res;
4818 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4819 N->getPointerInfo(), MachineMemOperand::MOStore,
4821 MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4822
4823 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
4824 N->getAddressingMode(), N->isTruncatingStore(),
4825 N->isCompressingStore());
4826
4827 if (HiIsEmpty) {
4828 // The hi masked store has zero storage size.
4829 // Only the lo masked store is needed.
4830 Res = Lo;
4831 } else {
4832
4833 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
4834 N->isCompressingStore());
4835
4836 MachinePointerInfo MPI;
4837 if (LoMemVT.isScalableVector()) {
4839 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4840 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4841 } else
4842 MPI = N->getPointerInfo().getWithOffset(
4843 LoMemVT.getStoreSize().getFixedValue());
4844
4845 MMO = DAG.getMachineFunction().getMachineMemOperand(
4847 Alignment,
4848 MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4849
4850 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
4851 N->getAddressingMode(), N->isTruncatingStore(),
4852 N->isCompressingStore());
4853
4854 // Build a factor node to remember that this store is independent of the
4855 // other one.
4856 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
4857 }
4858
4859 return Res;
4860}
4861
4862SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4863 SDValue Ch = N->getChain();
4864 SDValue Ptr = N->getBasePtr();
4865 EVT MemoryVT = N->getMemoryVT();
4866 Align Alignment = N->getBaseAlign();
4867 SDLoc DL(N);
4868 struct Operands {
4869 SDValue Mask;
4870 SDValue Index;
4871 SDValue Scale;
4872 SDValue Data;
4873 } Ops = [&]() -> Operands {
4874 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
4875 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
4876 MSC->getValue()};
4877 }
4878 auto *VPSC = cast<VPScatterSDNode>(N);
4879 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
4880 VPSC->getValue()};
4881 }();
4882 // Split all operands
4883
4884 EVT LoMemVT, HiMemVT;
4885 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
4886
4887 SDValue DataLo, DataHi;
4888 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4889 // Split Data operand
4890 GetSplitVector(Ops.Data, DataLo, DataHi);
4891 else
4892 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
4893
4894 // Split Mask operand
4895 SDValue MaskLo, MaskHi;
4896 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4897 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
4898 } else {
4899 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
4900 }
4901
4902 SDValue IndexHi, IndexLo;
4903 if (getTypeAction(Ops.Index.getValueType()) ==
4905 GetSplitVector(Ops.Index, IndexLo, IndexHi);
4906 else
4907 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
4908
4909 SDValue Lo;
4910 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4911 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4912 N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
4913 Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
4914
4915 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
4916 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4917 Lo =
4918 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
4919 MSC->getIndexType(), MSC->isTruncatingStore());
4920
4921 // The order of the Scatter operation after split is well defined. The "Hi"
4922 // part comes after the "Lo". So these two operations should be chained one
4923 // after another.
4924 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4925 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
4926 MMO, MSC->getIndexType(),
4927 MSC->isTruncatingStore());
4928 }
4929 auto *VPSC = cast<VPScatterSDNode>(N);
4930 SDValue EVLLo, EVLHi;
4931 std::tie(EVLLo, EVLHi) =
4932 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
4933
4934 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4935 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
4936 VPSC->getIndexType());
4937
4938 // The order of the Scatter operation after split is well defined. The "Hi"
4939 // part comes after the "Lo". So these two operations should be chained one
4940 // after another.
4941 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4942 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
4943 VPSC->getIndexType());
4944}
4945
4946SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4947 assert(N->isUnindexed() && "Indexed store of vector?");
4948 assert(OpNo == 1 && "Can only split the stored value");
4949 SDLoc DL(N);
4950
4951 bool isTruncating = N->isTruncatingStore();
4952 SDValue Ch = N->getChain();
4953 SDValue Ptr = N->getBasePtr();
4954 EVT MemoryVT = N->getMemoryVT();
4955 Align Alignment = N->getBaseAlign();
4956 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4957 MMOMetadata Metadata = N->getMMOMetadataForSubAccess();
4958 SDValue Lo, Hi;
4959 GetSplitVector(N->getOperand(1), Lo, Hi);
4960
4961 EVT LoMemVT, HiMemVT;
4962 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
4963
4964 // Scalarize if the split halves are not byte-sized.
4965 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4966 return TLI.scalarizeVectorStore(N, DAG);
4967
4968 if (isTruncating)
4969 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
4970 Alignment, MMOFlags, Metadata);
4971 else
4972 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
4973 Metadata);
4974
4975 MachinePointerInfo MPI;
4976 IncrementPointer(N, LoMemVT, MPI, Ptr);
4977
4978 if (isTruncating)
4979 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI, HiMemVT, Alignment, MMOFlags,
4980 Metadata);
4981 else
4982 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, Metadata);
4983
4984 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
4985}
4986
4987SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4988 SDLoc DL(N);
4989 LLVMContext &Ctx = *DAG.getContext();
4990 SDValue StVal = N->getVal();
4991 EVT VT = StVal.getValueType();
4992 EVT MemIntVT = EVT::getIntegerVT(Ctx, N->getMemoryVT().getSizeInBits());
4993
4994 // The store needs a single value spanning the full memory width. If the
4995 // value can be held in a legal vector register, keep it there and extract
4996 // the low integer element of the memory width. This lets the store be issued
4997 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4998 // bitcasting the split vector straight to a scalar integer, which would
4999 // reassemble the value element by element in GPRs.
5000 //
5001 // Reinterpret the value as a same-shaped integer vector first: an FP element
5002 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
5003 // integer-of-element-size form does. Ask the target which legal vector type
5004 // it widens to.
5005 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
5006 EVT IntEltVT = IntVecVT.getVectorElementType();
5007 EVT WideVT = TLI.getLegalTypeToTransformTo(Ctx, IntVecVT);
5008 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(MemIntVT) &&
5009 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
5010 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
5011 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
5012 SDValue Wide = ModifyToType(DAG.getBitcast(IntVecVT, StVal), WideVT);
5013 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
5014 EVT MemVecVT = EVT::getVectorVT(Ctx, MemIntVT, NumMemElts);
5015 SDValue Elt = DAG.getExtractVectorElt(DL, MemIntVT,
5016 DAG.getBitcast(MemVecVT, Wide), 0);
5017 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, MemIntVT, N->getChain(), Elt,
5018 N->getBasePtr(), N->getMemOperand());
5019 }
5020
5021 // Otherwise issue a single atomic store of an integer that spans the full
5022 // memory width. Bitcasting the (illegal) vector value to that integer lets
5023 // the type legalizer further legalize the BITCAST input as needed, while the
5024 // ATOMIC_STORE itself uses only the legal integer type.
5025 EVT IntVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits());
5026 SDValue AsInt = DAG.getBitcast(IntVT, StVal);
5027 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, MemIntVT, N->getChain(), AsInt,
5028 N->getBasePtr(), N->getMemOperand());
5029}
5030
5031SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
5032 SDLoc DL(N);
5033
5034 // The input operands all must have the same type, and we know the result
5035 // type is valid. Convert this to a buildvector which extracts all the
5036 // input elements.
5037 // TODO: If the input elements are power-two vectors, we could convert this to
5038 // a new CONCAT_VECTORS node with elements that are half-wide.
5040 EVT EltVT = N->getValueType(0).getVectorElementType();
5041 for (const SDValue &Op : N->op_values()) {
5042 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
5043 i != e; ++i) {
5044 Elts.push_back(DAG.getExtractVectorElt(DL, EltVT, Op, i));
5045 }
5046 }
5047
5048 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
5049}
5050
5051SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
5052 // The result type is legal, but the input type is illegal. If splitting
5053 // ends up with the result type of each half still being legal, just
5054 // do that. If, however, that would result in an illegal result type,
5055 // we can try to get more clever with power-two vectors. Specifically,
5056 // split the input type, but also widen the result element size, then
5057 // concatenate the halves and truncate again. For example, consider a target
5058 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
5059 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
5060 // %inlo = v4i32 extract_subvector %in, 0
5061 // %inhi = v4i32 extract_subvector %in, 4
5062 // %lo16 = v4i16 trunc v4i32 %inlo
5063 // %hi16 = v4i16 trunc v4i32 %inhi
5064 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
5065 // %res = v8i8 trunc v8i16 %in16
5066 //
5067 // Without this transform, the original truncate would end up being
5068 // scalarized, which is pretty much always a last resort.
5069 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
5070 SDValue InVec = N->getOperand(OpNo);
5071 EVT InVT = InVec->getValueType(0);
5072 EVT OutVT = N->getValueType(0);
5073 ElementCount NumElements = OutVT.getVectorElementCount();
5074 bool IsFloat = OutVT.isFloatingPoint();
5075
5076 unsigned InElementSize = InVT.getScalarSizeInBits();
5077 unsigned OutElementSize = OutVT.getScalarSizeInBits();
5078
5079 // Determine the split output VT. If its legal we can just split dirctly.
5080 EVT LoOutVT, HiOutVT;
5081 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
5082 assert(LoOutVT == HiOutVT && "Unequal split?");
5083
5084 // If the input elements are only 1/2 the width of the result elements,
5085 // just use the normal splitting. Our trick only work if there's room
5086 // to split more than once.
5087 if (isTypeLegal(LoOutVT) || InElementSize <= OutElementSize * 2 ||
5088 (IsFloat && !isPowerOf2_32(InElementSize)))
5089 return SplitVecOp_UnaryOp(N);
5090 SDLoc DL(N);
5091
5092 // Don't touch if this will be scalarized.
5093 EVT FinalVT = InVT;
5094 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
5095 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
5096
5097 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
5098 return SplitVecOp_UnaryOp(N);
5099
5100 // Get the split input vector.
5101 SDValue InLoVec, InHiVec;
5102 GetSplitVector(InVec, InLoVec, InHiVec);
5103
5104 // Truncate them to 1/2 the element size.
5105 //
5106 // This assumes the number of elements is a power of two; any vector that
5107 // isn't should be widened, not split.
5108 EVT HalfElementVT = IsFloat ?
5109 EVT::getFloatingPointVT(InElementSize/2) :
5110 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
5111 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
5112 NumElements.divideCoefficientBy(2));
5113
5114 SDValue HalfLo;
5115 SDValue HalfHi;
5116 SDValue Chain;
5117 if (N->isStrictFPOpcode()) {
5118 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
5119 {N->getOperand(0), InLoVec});
5120 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
5121 {N->getOperand(0), InHiVec});
5122 // Legalize the chain result - switch anything that used the old chain to
5123 // use the new one.
5124 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
5125 HalfHi.getValue(1));
5126 } else {
5127 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
5128 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
5129 }
5130
5131 // Concatenate them to get the full intermediate truncation result.
5132 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
5133 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
5134 HalfHi);
5135 // Now finish up by truncating all the way down to the original result
5136 // type. This should normally be something that ends up being legal directly,
5137 // but in theory if a target has very wide vectors and an annoyingly
5138 // restricted set of legal types, this split can chain to build things up.
5139
5140 if (N->isStrictFPOpcode()) {
5141 SDValue Res = DAG.getNode(
5142 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
5143 {Chain, InterVec,
5144 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
5145 // Relink the chain
5146 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
5147 return Res;
5148 }
5149
5150 return IsFloat
5151 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
5152 DAG.getTargetConstant(
5153 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
5154 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
5155}
5156
5157SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
5158 unsigned Opc = N->getOpcode();
5159 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
5160 assert(N->getValueType(0).isVector() &&
5161 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
5162 "Operand types must be vectors");
5163 // The result has a legal vector type, but the input needs splitting.
5164 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
5165 SDLoc DL(N);
5166 GetSplitVector(N->getOperand(isStrict ? 1 : 0), Lo0, Hi0);
5167 GetSplitVector(N->getOperand(isStrict ? 2 : 1), Lo1, Hi1);
5168
5169 EVT VT = N->getValueType(0);
5170 EVT PartResVT = getSetCCResultType(Lo0.getValueType());
5171
5172 if (Opc == ISD::SETCC) {
5173 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
5174 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
5175 } else {
5176 assert(isStrict && "unexpected node");
5177 LoRes = DAG.getNode(Opc, DL, DAG.getVTList(PartResVT, N->getValueType(1)),
5178 N->getOperand(0), Lo0, Lo1, N->getOperand(3));
5179 HiRes = DAG.getNode(Opc, DL, DAG.getVTList(PartResVT, N->getValueType(1)),
5180 N->getOperand(0), Hi0, Hi1, N->getOperand(3));
5181 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
5182 LoRes.getValue(1), HiRes.getValue(1));
5183 ReplaceValueWith(SDValue(N, 1), NewChain);
5184 }
5185
5186 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(*DAG.getContext());
5187 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, LoRes, HiRes);
5188 if (VT == ConcatVT)
5189 return Con;
5190
5191 EVT OpVT = N->getOperand(0).getValueType();
5192 ISD::NodeType ExtendCode =
5193 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
5194 return DAG.getExtOrTrunc(Con, DL, VT, ExtendCode);
5195}
5196
5197
5198SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
5199 // The result has a legal vector type, but the input needs splitting.
5200 EVT ResVT = N->getValueType(0);
5201 SDValue Lo, Hi;
5202 SDLoc DL(N);
5203 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
5204 EVT InVT = Lo.getValueType();
5205
5206 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
5207 InVT.getVectorElementCount());
5208
5209 if (N->isStrictFPOpcode()) {
5210 Lo = DAG.getNode(N->getOpcode(), DL, {OutVT, MVT::Other},
5211 {N->getOperand(0), Lo, N->getOperand(2)});
5212 Hi = DAG.getNode(N->getOpcode(), DL, {OutVT, MVT::Other},
5213 {N->getOperand(0), Hi, N->getOperand(2)});
5214 // Legalize the chain result - switch anything that used the old chain to
5215 // use the new one.
5216 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
5217 Lo.getValue(1), Hi.getValue(1));
5218 ReplaceValueWith(SDValue(N, 1), NewChain);
5219 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
5220 Lo = DAG.getNode(N->getOpcode(), DL, OutVT, Lo, N->getOperand(1),
5221 N->getOperand(2), N->getOperand(3));
5222 Hi = DAG.getNode(N->getOpcode(), DL, OutVT, Hi, N->getOperand(1),
5223 N->getOperand(2), N->getOperand(3));
5224 } else {
5225 Lo = DAG.getNode(N->getOpcode(), DL, OutVT, Lo, N->getOperand(1));
5226 Hi = DAG.getNode(N->getOpcode(), DL, OutVT, Hi, N->getOperand(1));
5227 }
5228
5229 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
5230}
5231
5232// Split a vector type in an FP binary operation where the second operand has a
5233// different type from the first.
5234//
5235// The result (and the first input) has a legal vector type, but the second
5236// input needs splitting.
5237SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5238 SDLoc DL(N);
5239
5240 EVT LHSLoVT, LHSHiVT;
5241 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
5242
5243 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
5244 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
5245
5246 SDValue LHSLo, LHSHi;
5247 std::tie(LHSLo, LHSHi) =
5248 DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT);
5249
5250 SDValue RHSLo, RHSHi;
5251 std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL);
5252
5253 SDValue Lo = DAG.getNode(N->getOpcode(), DL, LHSLoVT, LHSLo, RHSLo);
5254 SDValue Hi = DAG.getNode(N->getOpcode(), DL, LHSHiVT, LHSHi, RHSHi);
5255
5256 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
5257}
5258
5259SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5260 LLVMContext &Ctxt = *DAG.getContext();
5261 SDLoc dl(N);
5262
5263 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5264 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
5265 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
5266
5267 EVT ResVT = N->getValueType(0);
5268 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5269 EVT NewResVT =
5270 EVT::getVectorVT(Ctxt, ResVT.getVectorElementType(), SplitOpEC);
5271
5272 SDValue Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, LHSLo, RHSLo);
5273 SDValue Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, LHSHi, RHSHi);
5274
5275 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
5276}
5277
5278SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5279 EVT ResVT = N->getValueType(0);
5280 SDValue Lo, Hi;
5281 SDLoc dl(N);
5282 GetSplitVector(N->getOperand(0), Lo, Hi);
5283 EVT InVT = Lo.getValueType();
5284
5285 EVT NewResVT =
5286 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
5287 InVT.getVectorElementCount());
5288
5289 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
5290 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
5291
5292 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
5293}
5294
5295SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5296 SDLoc DL(N);
5297 EVT ResVT = N->getValueType(0);
5298
5299 SDValue Lo, Hi;
5300 SDValue VecOp = N->getOperand(0);
5301 GetSplitVector(VecOp, Lo, Hi);
5302
5303 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5304 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5305 SDValue ResLo = DAG.getNode(ISD::CTTZ_ELTS, DL, ResVT, Lo);
5306 SDValue VL =
5307 DAG.getElementCount(DL, ResVT, Lo.getValueType().getVectorElementCount());
5308 SDValue ResLoNotVL =
5309 DAG.getSetCC(DL, getSetCCResultType(ResVT), ResLo, VL, ISD::SETNE);
5310 SDValue ResHi = DAG.getNode(N->getOpcode(), DL, ResVT, Hi);
5311 return DAG.getSelect(DL, ResVT, ResLoNotVL, ResLo,
5312 DAG.getNode(ISD::ADD, DL, ResVT, VL, ResHi));
5313}
5314
5315SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5316 SDLoc DL(N);
5317 EVT ResVT = N->getValueType(0);
5318
5319 SDValue Lo, Hi;
5320 SDValue VecOp = N->getOperand(0);
5321 GetSplitVector(VecOp, Lo, Hi);
5322
5323 auto [MaskLo, MaskHi] = SplitMask(N->getOperand(1));
5324 auto [EVLLo, EVLHi] =
5325 DAG.SplitEVL(N->getOperand(2), VecOp.getValueType(), DL);
5326 SDValue VLo = DAG.getZExtOrTrunc(EVLLo, DL, ResVT);
5327
5328 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5329 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5330 SDValue ResLo = DAG.getNode(ISD::VP_CTTZ_ELTS, DL, ResVT, Lo, MaskLo, EVLLo);
5331 SDValue ResLoNotEVL =
5332 DAG.getSetCC(DL, getSetCCResultType(ResVT), ResLo, VLo, ISD::SETNE);
5333 SDValue ResHi = DAG.getNode(N->getOpcode(), DL, ResVT, Hi, MaskHi, EVLHi);
5334 return DAG.getSelect(DL, ResVT, ResLoNotEVL, ResLo,
5335 DAG.getNode(ISD::ADD, DL, ResVT, VLo, ResHi));
5336}
5337
5338SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5339 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(N);
5340 SDLoc DL(HG);
5341 SDValue Inc = HG->getInc();
5342 SDValue Ptr = HG->getBasePtr();
5343 SDValue Scale = HG->getScale();
5344 SDValue IntID = HG->getIntID();
5345 EVT MemVT = HG->getMemoryVT();
5346 MachineMemOperand *MMO = HG->getMemOperand();
5347 ISD::MemIndexType IndexType = HG->getIndexType();
5348
5349 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5350 std::tie(IndexLo, IndexHi) = DAG.SplitVector(HG->getIndex(), DL);
5351 std::tie(MaskLo, MaskHi) = DAG.SplitVector(HG->getMask(), DL);
5352 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5353 SDValue Lo = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT, DL,
5354 OpsLo, MMO, IndexType);
5355 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5356 return DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT, DL, OpsHi,
5357 MMO, IndexType);
5358}
5359
5360SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
5361 SDLoc DL(N);
5362
5363 if (OpNo == 0) {
5364 EVT LoResVT, HiResVT;
5365 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(N->getValueType(0));
5366 SDValue SourceLo, SourceHi;
5367 std::tie(SourceLo, SourceHi) = DAG.SplitVectorOperand(N, 0);
5368 SDValue MaskLo, MaskHi;
5369 std::tie(MaskLo, MaskHi) = DAG.SplitVectorOperand(N, 2);
5370
5371 SDValue MatchLo = DAG.getNode(ISD::VECTOR_MATCH, DL, LoResVT, SourceLo,
5372 N->getOperand(1), MaskLo, N->getFlags());
5373 SDValue MatchHi = DAG.getNode(ISD::VECTOR_MATCH, DL, HiResVT, SourceHi,
5374 N->getOperand(1), MaskHi, N->getFlags());
5375 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), MatchLo,
5376 MatchHi);
5377 }
5378
5379 // Note: The Mask (OpNo == 2) should be widened with the result.
5380 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
5381
5382 SDValue NeedleLo, NeedleHi;
5383 GetSplitVector(N->getOperand(1), NeedleLo, NeedleHi);
5384
5385 SDValue MatchLo =
5386 DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0), N->getOperand(0),
5387 NeedleLo, N->getOperand(2), N->getFlags());
5388 SDValue MatchHi =
5389 DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0), N->getOperand(0),
5390 NeedleHi, N->getOperand(2), N->getFlags());
5391 return DAG.getNode(ISD::OR, DL, N->getValueType(0), MatchLo, MatchHi);
5392}
5393
5394SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5395 SDValue Acc = N->getOperand(0);
5396 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5397 "Accumulator should already be a legal type, and shouldn't need "
5398 "further splitting");
5399
5400 SDLoc DL(N);
5401 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5402 GetSplitVector(N->getOperand(1), Input1Lo, Input1Hi);
5403 GetSplitVector(N->getOperand(2), Input2Lo, Input2Hi);
5404 unsigned Opcode = N->getOpcode();
5405 EVT ResultVT = Acc.getValueType();
5406
5407 SDValue Lo = DAG.getNode(Opcode, DL, ResultVT, Acc, Input1Lo, Input2Lo);
5408 return DAG.getNode(Opcode, DL, ResultVT, Lo, Input1Hi, Input2Hi);
5409}
5410
5411//===----------------------------------------------------------------------===//
5412// Result Vector Widening
5413//===----------------------------------------------------------------------===//
5414
5415void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5416 unsigned WidenResNo) {
5417 unsigned NumResults = N->getNumValues();
5418 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5419 if (ResNo == WidenResNo)
5420 continue;
5421 EVT ResVT = N->getValueType(ResNo);
5422 if (getTypeAction(ResVT) == TargetLowering::TypeWidenVector) {
5423 SetWidenedVector(SDValue(N, ResNo), SDValue(WidenNode, ResNo));
5424 } else {
5425 SDLoc DL(N);
5426 SDValue ResVal =
5427 DAG.getExtractSubvector(DL, ResVT, SDValue(WidenNode, ResNo), 0);
5428 ReplaceValueWith(SDValue(N, ResNo), ResVal);
5429 }
5430 }
5431}
5432
5433void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5434 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5435
5436 // See if the target wants to custom widen this node.
5437 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
5438 return;
5439
5440 SDValue Res = SDValue();
5441
5442 auto unrollExpandedOp = [&]() {
5443 // We're going to widen this vector op to a legal type by padding with undef
5444 // elements. If the wide vector op is eventually going to be expanded to
5445 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5446 // libcalls on the undef elements.
5447
5448 // The lane count comes from the result being widened, but operation
5449 // legality is queried using the corresponding result 0 type.
5450 EVT ResVT = N->getValueType(ResNo);
5451 EVT WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
5452 if (WideResVT.isScalableVector())
5453 return false;
5454
5455 EVT VT0 = N->getValueType(0);
5456 EVT WideVecVT = VT0.changeVectorElementCount(
5457 *DAG.getContext(), WideResVT.getVectorElementCount());
5458
5459 if (!TLI.isOperationLegalOrCustomOrPromote(N->getOpcode(), WideVecVT) &&
5460 TLI.isOperationExpandOrLibCall(N->getOpcode(), VT0.getScalarType())) {
5461 SDValue Unrolled =
5462 DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
5463 Res = Unrolled.getValue(ResNo);
5464 if (N->getNumValues() > 1)
5465 ReplaceOtherWidenResults(N, Unrolled.getNode(), ResNo);
5466 return true;
5467 }
5468 return false;
5469 };
5470
5471 switch (N->getOpcode()) {
5472 default:
5473#ifndef NDEBUG
5474 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5475 N->dump(&DAG);
5476 dbgs() << "\n";
5477#endif
5478 report_fatal_error("do not know how to widen the result of this operator!");
5479
5482 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5483 break;
5484 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5485 case ISD::ADDRSPACECAST:
5486 Res = WidenVecRes_ADDRSPACECAST(N);
5487 break;
5488 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5489 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5490 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5491 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5493 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5494 break;
5495 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5496 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5497 case ISD::ATOMIC_LOAD:
5498 Res = WidenVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
5499 break;
5500 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5501 case ISD::STEP_VECTOR:
5502 case ISD::SPLAT_VECTOR:
5504 Res = WidenVecRes_ScalarOp(N);
5505 break;
5506 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5507 case ISD::VSELECT:
5508 case ISD::SELECT:
5509 case ISD::VP_MERGE:
5510 Res = WidenVecRes_Select(N);
5511 break;
5512 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5513 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5514 case ISD::POISON:
5515 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5517 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
5518 break;
5519 case ISD::VP_LOAD:
5520 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
5521 break;
5522 case ISD::VP_LOAD_FF:
5523 Res = WidenVecRes_VP_LOAD_FF(cast<VPLoadFFSDNode>(N));
5524 break;
5525 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5526 Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N));
5527 break;
5529 Res = WidenVecRes_VECTOR_COMPRESS(N);
5530 break;
5532 Res = WidenVecRes_VECTOR_SHUFFLE_VAR(N);
5533 break;
5534 case ISD::MLOAD:
5535 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
5536 break;
5537 case ISD::MGATHER:
5538 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
5539 break;
5540 case ISD::VP_GATHER:
5541 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
5542 break;
5544 Res = WidenVecRes_VECTOR_REVERSE(N);
5545 break;
5547 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5548 break;
5550 WidenVecRes_VECTOR_INTERLEAVE(N);
5551 break;
5552 case ISD::VECTOR_MATCH:
5553 Res = WidenVecRes_VECTOR_MATCH(N);
5554 break;
5556 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5557 break;
5558
5559 case ISD::ADD:
5560 case ISD::AND:
5561 case ISD::MUL:
5562 case ISD::MULHS:
5563 case ISD::MULHU:
5564 case ISD::ABDS:
5565 case ISD::ABDU:
5566 case ISD::OR:
5567 case ISD::SUB:
5568 case ISD::XOR:
5569 case ISD::SHL:
5570 case ISD::SRA:
5571 case ISD::SRL:
5572 case ISD::CLMUL:
5573 case ISD::CLMULR:
5574 case ISD::CLMULH:
5575 case ISD::PEXT:
5576 case ISD::PDEP:
5577 case ISD::FMINNUM:
5578 case ISD::FMINNUM_IEEE:
5579 case ISD::FMAXNUM:
5580 case ISD::FMAXNUM_IEEE:
5581 case ISD::FMINIMUM:
5582 case ISD::FMAXIMUM:
5583 case ISD::FMINIMUMNUM:
5584 case ISD::FMAXIMUMNUM:
5585 case ISD::SMIN:
5586 case ISD::SMAX:
5587 case ISD::UMIN:
5588 case ISD::UMAX:
5589 case ISD::UADDSAT:
5590 case ISD::SADDSAT:
5591 case ISD::USUBSAT:
5592 case ISD::SSUBSAT:
5593 case ISD::SSHLSAT:
5594 case ISD::USHLSAT:
5595 case ISD::ROTL:
5596 case ISD::ROTR:
5597 case ISD::AVGFLOORS:
5598 case ISD::AVGFLOORU:
5599 case ISD::AVGCEILS:
5600 case ISD::AVGCEILU:
5601 case ISD::AVGSUBFLOORS:
5602 case ISD::AVGSUBFLOORU:
5603 // Vector-predicated binary op widening. Note that -- unlike the
5604 // unpredicated versions -- we don't have to worry about trapping on
5605 // operations like UDIV, FADD, etc., as we pass on the original vector
5606 // length parameter. This means the widened elements containing garbage
5607 // aren't active.
5608 case ISD::VP_SDIV:
5609 case ISD::VP_UDIV:
5610 case ISD::VP_SREM:
5611 case ISD::VP_UREM:
5612 Res = WidenVecRes_Binary(N);
5613 break;
5614
5615 case ISD::MASKED_UDIV:
5616 case ISD::MASKED_SDIV:
5617 case ISD::MASKED_UREM:
5618 case ISD::MASKED_SREM:
5619 Res = WidenVecRes_MaskedBinary(N);
5620 break;
5621
5622 case ISD::SCMP:
5623 case ISD::UCMP:
5624 Res = WidenVecRes_CMP(N);
5625 break;
5626
5627 case ISD::FPOW:
5628 case ISD::FATAN2:
5629 case ISD::FREM:
5630 if (unrollExpandedOp())
5631 break;
5632 // If the target has custom/legal support for the scalar FP intrinsic ops
5633 // (they are probably not destined to become libcalls), then widen those
5634 // like any other binary ops.
5635 [[fallthrough]];
5636
5637 case ISD::FADD:
5638 case ISD::FMUL:
5639 case ISD::FSUB:
5640 case ISD::FDIV:
5641 case ISD::SDIV:
5642 case ISD::UDIV:
5643 case ISD::SREM:
5644 case ISD::UREM:
5645 Res = WidenVecRes_BinaryCanTrap(N);
5646 break;
5647
5648 case ISD::SMULFIX:
5649 case ISD::SMULFIXSAT:
5650 case ISD::UMULFIX:
5651 case ISD::UMULFIXSAT:
5652 // These are binary operations, but with an extra operand that shouldn't
5653 // be widened (the scale).
5654 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5655 break;
5656
5657#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5658 case ISD::STRICT_##DAGN:
5659#include "llvm/IR/ConstrainedOps.def"
5660 Res = WidenVecRes_StrictFP(N);
5661 break;
5662
5663 case ISD::UADDO:
5664 case ISD::SADDO:
5665 case ISD::USUBO:
5666 case ISD::SSUBO:
5667 case ISD::UMULO:
5668 case ISD::SMULO:
5669 Res = WidenVecRes_OverflowOp(N, ResNo);
5670 break;
5671
5672 case ISD::FCOPYSIGN:
5673 Res = WidenVecRes_FCOPYSIGN(N);
5674 break;
5675
5676 case ISD::IS_FPCLASS:
5677 case ISD::FPTRUNC_ROUND:
5678 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5679 break;
5680
5681 case ISD::FLDEXP:
5682 case ISD::FPOWI:
5683 if (!unrollExpandedOp())
5684 Res = WidenVecRes_ExpOp(N);
5685 break;
5686
5690 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5691 break;
5692
5693 case ISD::ANY_EXTEND:
5694 case ISD::FP_EXTEND:
5695 case ISD::FP_ROUND:
5696 case ISD::FP_TO_SINT:
5697 case ISD::FP_TO_UINT:
5698 case ISD::SIGN_EXTEND:
5699 case ISD::SINT_TO_FP:
5700 case ISD::TRUNCATE:
5701 case ISD::UINT_TO_FP:
5702 case ISD::ZERO_EXTEND:
5705 Res = WidenVecRes_Convert(N);
5706 break;
5707
5710 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5711 break;
5712
5713 case ISD::LRINT:
5714 case ISD::LLRINT:
5715 case ISD::LROUND:
5716 case ISD::LLROUND:
5717 Res = WidenVecRes_XROUND(N);
5718 break;
5719
5720 case ISD::FACOS:
5721 case ISD::FASIN:
5722 case ISD::FATAN:
5723 case ISD::FCEIL:
5724 case ISD::FCOS:
5725 case ISD::FCOSH:
5726 case ISD::FEXP:
5727 case ISD::FEXP2:
5728 case ISD::FEXP10:
5729 case ISD::FFLOOR:
5730 case ISD::FLOG:
5731 case ISD::FLOG10:
5732 case ISD::FLOG2:
5733 case ISD::FNEARBYINT:
5734 case ISD::FRINT:
5735 case ISD::FROUND:
5736 case ISD::FROUNDEVEN:
5737 case ISD::FSIN:
5738 case ISD::FSINH:
5739 case ISD::FSQRT:
5740 case ISD::FTAN:
5741 case ISD::FTANH:
5742 case ISD::FTRUNC:
5743 if (unrollExpandedOp())
5744 break;
5745 // If the target has custom/legal support for the scalar FP intrinsic ops
5746 // (they are probably not destined to become libcalls), then widen those
5747 // like any other unary ops.
5748 [[fallthrough]];
5749
5750 case ISD::ABS:
5752 case ISD::BITREVERSE:
5753 case ISD::BSWAP:
5754 case ISD::CTLZ:
5756 case ISD::CTPOP:
5757 case ISD::CTTZ:
5759 case ISD::FNEG:
5760 case ISD::FABS:
5761 case ISD::FREEZE:
5762 case ISD::ARITH_FENCE:
5763 case ISD::FCANONICALIZE:
5766 Res = WidenVecRes_Unary(N);
5767 break;
5768 case ISD::FMA:
5769 case ISD::FSHL:
5770 case ISD::FSHR:
5771 Res = WidenVecRes_Ternary(N);
5772 break;
5773 case ISD::FMODF:
5774 case ISD::FFREXP:
5775 case ISD::FSINCOS:
5776 case ISD::FSINCOSPI: {
5777 if (!unrollExpandedOp())
5778 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5779 break;
5780 }
5785 Res = WidenVecRes_PARTIAL_REDUCE_MLA(N);
5786 break;
5787 }
5788
5789 // If Res is null, the sub-method took care of registering the result.
5790 if (Res.getNode())
5791 SetWidenedVector(SDValue(N, ResNo), Res);
5792}
5793
5794SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5795 // Ternary op widening.
5796 SDLoc dl(N);
5797 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5798 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5799 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5800 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
5801 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
5802}
5803
5804SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5805 // Binary op widening.
5806 SDLoc dl(N);
5807 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5808 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5809 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5810 if (N->getNumOperands() == 2)
5811 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
5812 N->getFlags());
5813
5814 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5815 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
5816 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
5817 "Expected VP opcode");
5818
5819 SDValue Mask =
5820 GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount());
5821 return DAG.getNode(N->getOpcode(), dl, WidenVT,
5822 {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
5823}
5824
5825SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5826 SDLoc dl(N);
5827 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5828 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5829 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5830 SDValue Mask = N->getOperand(2);
5831 EVT WideMaskVT = WidenVT.changeVectorElementType(
5832 *DAG.getContext(), Mask.getValueType().getVectorElementType());
5833 Mask = ModifyToType(Mask, WideMaskVT, /*FillWithZeros=*/true);
5834 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Mask,
5835 N->getFlags());
5836}
5837
5838SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5839 LLVMContext &Ctxt = *DAG.getContext();
5840 SDLoc dl(N);
5841
5842 SDValue LHS = N->getOperand(0);
5843 SDValue RHS = N->getOperand(1);
5844 EVT OpVT = LHS.getValueType();
5845 if (getTypeAction(OpVT) == TargetLowering::TypeWidenVector) {
5846 LHS = GetWidenedVector(LHS);
5847 RHS = GetWidenedVector(RHS);
5848 OpVT = LHS.getValueType();
5849 }
5850
5851 EVT WidenResVT = TLI.getTypeToTransformTo(Ctxt, N->getValueType(0));
5852 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5853 if (WidenResEC == OpVT.getVectorElementCount()) {
5854 return DAG.getNode(N->getOpcode(), dl, WidenResVT, LHS, RHS);
5855 }
5856
5857 return DAG.UnrollVectorOp(N, WidenResVT.getVectorNumElements());
5858}
5859
5860SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5861 // Binary op widening, but with an extra operand that shouldn't be widened.
5862 SDLoc dl(N);
5863 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5864 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5865 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5866 SDValue InOp3 = N->getOperand(2);
5867 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
5868 N->getFlags());
5869}
5870
5871// Given a vector of operations that have been broken up to widen, see
5872// if we can collect them together into the next widest legal VT. This
5873// implementation is trap-safe.
5875 SmallVectorImpl<SDValue> &ConcatOps,
5876 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5877 EVT WidenVT) {
5878 // Check to see if we have a single operation with the widen type.
5879 if (ConcatEnd == 1) {
5880 VT = ConcatOps[0].getValueType();
5881 if (VT == WidenVT)
5882 return ConcatOps[0];
5883 }
5884
5885 SDLoc dl(ConcatOps[0]);
5886 EVT WidenEltVT = WidenVT.getVectorElementType();
5887
5888 // while (Some element of ConcatOps is not of type MaxVT) {
5889 // From the end of ConcatOps, collect elements of the same type and put
5890 // them into an op of the next larger supported type
5891 // }
5892 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5893 int Idx = ConcatEnd - 1;
5894 VT = ConcatOps[Idx--].getValueType();
5895 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5896 Idx--;
5897
5898 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5899 EVT NextVT;
5900 do {
5901 NextSize *= 2;
5902 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
5903 } while (!TLI.isTypeLegal(NextVT));
5904
5905 if (!VT.isVector()) {
5906 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5907 SDValue VecOp = DAG.getPOISON(NextVT);
5908 unsigned NumToInsert = ConcatEnd - Idx - 1;
5909 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5910 VecOp = DAG.getInsertVectorElt(dl, VecOp, ConcatOps[OpIdx], i);
5911 ConcatOps[Idx+1] = VecOp;
5912 ConcatEnd = Idx + 2;
5913 } else {
5914 // Vector type, create a CONCAT_VECTORS of type NextVT
5915 SDValue undefVec = DAG.getPOISON(VT);
5916 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5917 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5918 unsigned RealVals = ConcatEnd - Idx - 1;
5919 unsigned SubConcatEnd = 0;
5920 unsigned SubConcatIdx = Idx + 1;
5921 while (SubConcatEnd < RealVals)
5922 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5923 while (SubConcatEnd < OpsToConcat)
5924 SubConcatOps[SubConcatEnd++] = undefVec;
5925 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
5926 NextVT, SubConcatOps);
5927 ConcatEnd = SubConcatIdx + 1;
5928 }
5929 }
5930
5931 // Check to see if we have a single operation with the widen type.
5932 if (ConcatEnd == 1) {
5933 VT = ConcatOps[0].getValueType();
5934 if (VT == WidenVT)
5935 return ConcatOps[0];
5936 }
5937
5938 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5940 MaxVT.getVectorElementCount());
5941 if (NumOps != ConcatEnd ) {
5942 SDValue UndefVal = DAG.getPOISON(MaxVT);
5943 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5944 ConcatOps[j] = UndefVal;
5945 }
5946 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
5947 ArrayRef(ConcatOps.data(), NumOps));
5948}
5949
5950SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5951 // Binary op widening for operations that can trap.
5952 unsigned Opcode = N->getOpcode();
5953 SDLoc dl(N);
5954 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5955 EVT VT = WidenVT;
5956 ElementCount NumElts = VT.getVectorElementCount();
5957 const SDNodeFlags Flags = N->getFlags();
5958 while (!TLI.isTypeLegal(VT) && NumElts.getKnownMinValue() != 1) {
5959 NumElts = NumElts.divideCoefficientBy(2);
5960 VT = VT.changeVectorElementCount(*DAG.getContext(), NumElts);
5961 }
5962
5963 bool FoundLegalVectorType = TLI.isTypeLegal(VT) && !NumElts.isScalar();
5964 if (FoundLegalVectorType && DAG.isSafeToSpeculativelyExecute(Opcode)) {
5965 // Operation can be speculatively executed so just widen as normal.
5966 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5967 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5968 return DAG.getNode(Opcode, dl, WidenVT, InOp1, InOp2, Flags);
5969 }
5970
5971 // Generate a vp.op if it is custom/legal for the target. This avoids need
5972 // to split and tile the subvectors (below), because the inactive lanes can
5973 // simply be disabled. To avoid possible recursion, only do this if the
5974 // widened mask type is legal.
5975 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5976 VPOpcode && TLI.isOperationLegalOrCustom(*VPOpcode, WidenVT)) {
5977 if (EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5978 WidenVT.getVectorElementCount());
5979 TLI.isTypeLegal(WideMaskVT)) {
5980 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5981 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5982 SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
5983 SDValue EVL =
5984 DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
5985 N->getValueType(0).getVectorElementCount());
5986 return DAG.getNode(*VPOpcode, dl, WidenVT, InOp1, InOp2, Mask, EVL,
5987 Flags);
5988 }
5989 }
5990
5991 // No legal vector version so unroll the vector operation and then widen.
5992 if (NumElts.isScalar())
5993 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
5994
5995 if (!TLI.isTypeLegal(VT))
5996 report_fatal_error("Cannot widen scalable vector trapping binop!");
5997
5998 // Since the operation can trap, apply operation on the original vector.
5999 EVT MaxVT = VT;
6000 EVT EltVT = VT.getVectorElementType();
6001 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
6002 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
6003 ElementCount CurNumElts = N->getValueType(0).getVectorElementCount();
6004
6005 SmallVector<SDValue, 16> ConcatOps(CurNumElts.getKnownMinValue());
6006 unsigned ConcatEnd = 0; // Current ConcatOps index.
6007 int Idx = 0; // Current Idx into input vectors.
6008
6009 // NumElts := greatest legal vector size (at most WidenVT)
6010 // while (orig. vector has unhandled elements) {
6011 // take munches of size NumElts from the beginning and add to ConcatOps
6012 // NumElts := next smaller supported vector size or 1
6013 // }
6014 for (;;) {
6015 while (ElementCount::isKnownGE(CurNumElts, NumElts)) {
6016 SDValue EOp1 = DAG.getExtractSubvector(dl, VT, InOp1, Idx);
6017 SDValue EOp2 = DAG.getExtractSubvector(dl, VT, InOp2, Idx);
6018 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
6019 // Subvector indices are implicitly scaled by vscale.
6020 Idx += NumElts.getKnownMinValue();
6021 CurNumElts -= NumElts;
6022 }
6023
6024 if (CurNumElts.isZero())
6025 break;
6026
6027 // Find the next smaller legal vector type.
6028 do {
6029 NumElts = NumElts.divideCoefficientBy(2);
6030 VT = VT.changeVectorElementCount(*DAG.getContext(), NumElts);
6031 } while (!TLI.isTypeLegal(VT) && NumElts.getKnownMinValue() != 1);
6032
6033 if (NumElts.isScalar()) {
6034 // Scalarize the non-type-legal remainder.
6035 for (unsigned i = 0; i != CurNumElts.getFixedValue(); ++i, ++Idx) {
6036 SDValue EOp1 = DAG.getExtractVectorElt(dl, EltVT, InOp1, Idx);
6037 SDValue EOp2 = DAG.getExtractVectorElt(dl, EltVT, InOp2, Idx);
6038 ConcatOps[ConcatEnd++] =
6039 DAG.getNode(Opcode, dl, EltVT, EOp1, EOp2, Flags);
6040 }
6041
6042 break;
6043 }
6044
6045 if (!TLI.isTypeLegal(VT))
6046 report_fatal_error("scalarization of scalable vectors is not supported");
6047 }
6048
6049 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
6050}
6051
6052SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
6053 switch (N->getOpcode()) {
6054 case ISD::STRICT_FSETCC:
6056 return WidenVecRes_STRICT_FSETCC(N);
6063 return WidenVecRes_Convert_StrictFP(N);
6064 default:
6065 break;
6066 }
6067
6068 // StrictFP op widening for operations that can trap.
6069 unsigned NumOpers = N->getNumOperands();
6070 unsigned Opcode = N->getOpcode();
6071 SDLoc dl(N);
6072 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6073 EVT WidenEltVT = WidenVT.getVectorElementType();
6074 EVT VT = WidenVT;
6075 unsigned NumElts = VT.getVectorNumElements();
6076 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
6077 NumElts = NumElts / 2;
6078 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
6079 }
6080
6081 // No legal vector version so unroll the vector operation and then widen.
6082 if (NumElts == 1)
6083 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
6084
6085 // Since the operation can trap, apply operation on the original vector.
6086 EVT MaxVT = VT;
6088 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
6089
6090 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
6092 unsigned ConcatEnd = 0; // Current ConcatOps index.
6093 int Idx = 0; // Current Idx into input vectors.
6094
6095 // The Chain is the first operand.
6096 InOps.push_back(N->getOperand(0));
6097
6098 // Now process the remaining operands.
6099 for (unsigned i = 1; i < NumOpers; ++i) {
6100 SDValue Oper = N->getOperand(i);
6101
6102 EVT OpVT = Oper.getValueType();
6103 if (OpVT.isVector()) {
6104 if (getTypeAction(OpVT) == TargetLowering::TypeWidenVector)
6105 Oper = GetWidenedVector(Oper);
6106 else {
6107 EVT WideOpVT =
6108 EVT::getVectorVT(*DAG.getContext(), OpVT.getVectorElementType(),
6109 WidenVT.getVectorElementCount());
6110 Oper = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
6111 DAG.getPOISON(WideOpVT), Oper,
6112 DAG.getVectorIdxConstant(0, dl));
6113 }
6114 }
6115
6116 InOps.push_back(Oper);
6117 }
6118
6119 // NumElts := greatest legal vector size (at most WidenVT)
6120 // while (orig. vector has unhandled elements) {
6121 // take munches of size NumElts from the beginning and add to ConcatOps
6122 // NumElts := next smaller supported vector size or 1
6123 // }
6124 while (CurNumElts != 0) {
6125 while (CurNumElts >= NumElts) {
6127
6128 for (unsigned i = 0; i < NumOpers; ++i) {
6129 SDValue Op = InOps[i];
6130
6131 EVT OpVT = Op.getValueType();
6132 if (OpVT.isVector()) {
6133 EVT OpExtractVT =
6134 EVT::getVectorVT(*DAG.getContext(), OpVT.getVectorElementType(),
6136 Op = DAG.getExtractSubvector(dl, OpExtractVT, Op, Idx);
6137 }
6138
6139 EOps.push_back(Op);
6140 }
6141
6142 EVT OperVT[] = {VT, MVT::Other};
6143 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
6144 ConcatOps[ConcatEnd++] = Oper;
6145 Chains.push_back(Oper.getValue(1));
6146 Idx += NumElts;
6147 CurNumElts -= NumElts;
6148 }
6149 do {
6150 NumElts = NumElts / 2;
6151 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
6152 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
6153
6154 if (NumElts == 1) {
6155 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
6157
6158 for (unsigned i = 0; i < NumOpers; ++i) {
6159 SDValue Op = InOps[i];
6160
6161 EVT OpVT = Op.getValueType();
6162 if (OpVT.isVector())
6163 Op = DAG.getExtractVectorElt(dl, OpVT.getVectorElementType(), Op,
6164 Idx);
6165
6166 EOps.push_back(Op);
6167 }
6168
6169 EVT WidenVT[] = {WidenEltVT, MVT::Other};
6170 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
6171 ConcatOps[ConcatEnd++] = Oper;
6172 Chains.push_back(Oper.getValue(1));
6173 }
6174 CurNumElts = 0;
6175 }
6176 }
6177
6178 // Build a factor node to remember all the Ops that have been created.
6179 SDValue NewChain;
6180 if (Chains.size() == 1)
6181 NewChain = Chains[0];
6182 else
6183 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6184 ReplaceValueWith(SDValue(N, 1), NewChain);
6185
6186 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
6187}
6188
6189SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
6190 SDLoc DL(N);
6191 EVT ResVT = N->getValueType(0);
6192 EVT OvVT = N->getValueType(1);
6193 EVT WideResVT, WideOvVT;
6194 SDValue WideLHS, WideRHS;
6195
6196 // TODO: This might result in a widen/split loop.
6197 if (ResNo == 0) {
6198 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
6199 WideOvVT = EVT::getVectorVT(
6200 *DAG.getContext(), OvVT.getVectorElementType(),
6201 WideResVT.getVectorNumElements());
6202
6203 WideLHS = GetWidenedVector(N->getOperand(0));
6204 WideRHS = GetWidenedVector(N->getOperand(1));
6205 } else {
6206 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
6207 WideResVT = EVT::getVectorVT(
6208 *DAG.getContext(), ResVT.getVectorElementType(),
6209 WideOvVT.getVectorNumElements());
6210
6211 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
6212 SDValue Poison = DAG.getPOISON(WideResVT);
6213
6214 WideLHS = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideResVT, Poison,
6215 N->getOperand(0), Zero);
6216 WideRHS = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideResVT, Poison,
6217 N->getOperand(1), Zero);
6218 }
6219
6220 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
6221 SDNode *WideNode = DAG.getNode(
6222 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
6223
6224 // Replace the other vector result not being explicitly widened here.
6225 unsigned OtherNo = 1 - ResNo;
6226 EVT OtherVT = N->getValueType(OtherNo);
6227 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
6228 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
6229 } else {
6230 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
6231 SDValue OtherVal = DAG.getNode(
6232 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
6233 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
6234 }
6235
6236 return SDValue(WideNode, ResNo);
6237}
6238
6239SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
6240 LLVMContext &Ctx = *DAG.getContext();
6241 SDValue InOp = N->getOperand(0);
6242 SDLoc DL(N);
6243
6244 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
6245 ElementCount WidenEC = WidenVT.getVectorElementCount();
6246
6247 EVT InVT = InOp.getValueType();
6248
6249 unsigned Opcode = N->getOpcode();
6250 const SDNodeFlags Flags = N->getFlags();
6251
6252 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
6253 // not equal that of WidenVT.
6254 if (N->getOpcode() == ISD::ZERO_EXTEND &&
6255 getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
6256 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
6257 WidenVT.getScalarSizeInBits()) {
6258 InOp = ZExtPromotedInteger(InOp);
6259 InVT = InOp.getValueType();
6260 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
6261 Opcode = ISD::TRUNCATE;
6262 }
6263
6264 EVT InEltVT = InVT.getVectorElementType();
6265 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
6266 ElementCount InVTEC = InVT.getVectorElementCount();
6267
6268 // Helper to build node with all scalar trailing operands.
6269 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6270 if (N->getNumOperands() == 1)
6271 return DAG.getNode(Opcode, DL, VT, Op, Flags);
6272 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6273 return DAG.getNode(Opcode, DL, VT, Op, N->getOperand(1), N->getOperand(2),
6274 N->getOperand(3), Flags);
6275 return DAG.getNode(Opcode, DL, VT, Op, N->getOperand(1), Flags);
6276 };
6277
6278 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
6279 InOp = GetWidenedVector(N->getOperand(0));
6280 InVT = InOp.getValueType();
6281 InVTEC = InVT.getVectorElementCount();
6282 if (InVTEC == WidenEC)
6283 return MakeConvertNode(WidenVT, InOp);
6284 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6285 // If both input and result vector types are of same width, extend
6286 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6287 // accepts fewer elements in the result than in the input.
6288 if (Opcode == ISD::ANY_EXTEND)
6289 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
6290 if (Opcode == ISD::SIGN_EXTEND)
6291 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
6292 if (Opcode == ISD::ZERO_EXTEND)
6293 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
6294 }
6295
6296 // For TRUNCATE, try to widen using the legal EC of the input type instead
6297 // if the legalisation action for that intermediate type is not widening.
6298 // E.g. for trunc nxv1i64 -> nxv1i8 where
6299 // - nxv1i64 input gets widened to nxv2i64
6300 // - nxv1i8 output gets widened to nxv16i8
6301 // Then one can try widening the result to nxv2i8 (instead of going all the
6302 // way to nxv16i8) if this later allows type promotion.
6303 EVT MidResVT =
6304 EVT::getVectorVT(Ctx, WidenVT.getVectorElementType(), InVTEC);
6305 if (N->getOpcode() == ISD::TRUNCATE &&
6306 getTypeAction(MidResVT) == TargetLowering::TypePromoteInteger) {
6307 SDValue MidRes = DAG.getNode(ISD::TRUNCATE, DL, MidResVT, InOp, Flags);
6308 return DAG.getInsertSubvector(DL, DAG.getPOISON(WidenVT), MidRes, 0);
6309 }
6310 }
6311
6312 if (TLI.isTypeLegal(InWidenVT)) {
6313 // Because the result and the input are different vector types, widening
6314 // the result could create a legal type but widening the input might make
6315 // it an illegal type that might lead to repeatedly splitting the input
6316 // and then widening it. To avoid this, we widen the input only if
6317 // it results in a legal type.
6318 if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
6319 // Widen the input and call convert on the widened input vector.
6320 unsigned NumConcat =
6321 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6322 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(InVT));
6323 Ops[0] = InOp;
6324 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
6325 return MakeConvertNode(WidenVT, InVec);
6326 }
6327
6328 if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
6329 SDValue InVal = DAG.getExtractSubvector(DL, InWidenVT, InOp, 0);
6330 // Extract the input and convert the shorten input vector.
6331 return MakeConvertNode(WidenVT, InVal);
6332 }
6333 }
6334
6335 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6336 EVT EltVT = WidenVT.getVectorElementType();
6337 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(EltVT));
6338 // Use the original element count so we don't do more scalar opts than
6339 // necessary.
6340 unsigned MinElts = N->getValueType(0).getVectorNumElements();
6341 for (unsigned i=0; i < MinElts; ++i) {
6342 SDValue Val = DAG.getExtractVectorElt(DL, InEltVT, InOp, i);
6343 Ops[i] = MakeConvertNode(EltVT, Val);
6344 }
6345
6346 return DAG.getBuildVector(WidenVT, DL, Ops);
6347}
6348
6349SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6350 SDLoc dl(N);
6351 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6352 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6353
6354 SDValue Src = N->getOperand(0);
6355 EVT SrcVT = Src.getValueType();
6356
6357 // Also widen the input.
6358 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
6359 Src = GetWidenedVector(Src);
6360 SrcVT = Src.getValueType();
6361 }
6362
6363 // Input and output not widened to the same size, give up.
6364 if (WidenNumElts != SrcVT.getVectorElementCount())
6365 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
6366
6367 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
6368}
6369
6370SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6371 SDLoc dl(N);
6372 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6373 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6374
6375 SDValue Src = N->getOperand(0);
6376 EVT SrcVT = Src.getValueType();
6377
6378 // Also widen the input.
6379 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
6380 Src = GetWidenedVector(Src);
6381 SrcVT = Src.getValueType();
6382 }
6383
6384 // Input and output not widened to the same size, give up.
6385 if (WidenNumElts != SrcVT.getVectorElementCount())
6386 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
6387
6388 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src);
6389}
6390
6391SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6392 SDValue InOp = N->getOperand(1);
6393 SDLoc DL(N);
6394 SmallVector<SDValue, 4> NewOps(N->ops());
6395
6396 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6397 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6398
6399 EVT InVT = InOp.getValueType();
6400 EVT InEltVT = InVT.getVectorElementType();
6401
6402 unsigned Opcode = N->getOpcode();
6403
6404 // FIXME: Optimizations need to be implemented here.
6405
6406 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6407 EVT EltVT = WidenVT.getVectorElementType();
6408 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
6409 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(EltVT));
6410 SmallVector<SDValue, 32> OpChains;
6411 // Use the original element count so we don't do more scalar opts than
6412 // necessary.
6413 unsigned MinElts = N->getValueType(0).getVectorNumElements();
6414 for (unsigned i=0; i < MinElts; ++i) {
6415 NewOps[1] = DAG.getExtractVectorElt(DL, InEltVT, InOp, i);
6416 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
6417 OpChains.push_back(Ops[i].getValue(1));
6418 }
6419 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
6420 ReplaceValueWith(SDValue(N, 1), NewChain);
6421
6422 return DAG.getBuildVector(WidenVT, DL, Ops);
6423}
6424
6425SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6426 unsigned Opcode = N->getOpcode();
6427 SDValue InOp = N->getOperand(0);
6428 SDLoc DL(N);
6429
6430 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6431 EVT WidenSVT = WidenVT.getVectorElementType();
6432 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6433
6434 EVT InVT = InOp.getValueType();
6435 EVT InSVT = InVT.getVectorElementType();
6436 unsigned InVTNumElts = InVT.getVectorNumElements();
6437
6438 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
6439 InOp = GetWidenedVector(InOp);
6440 InVT = InOp.getValueType();
6441 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6442 switch (Opcode) {
6446 return DAG.getNode(Opcode, DL, WidenVT, InOp);
6447 }
6448 }
6449 }
6450
6451 // Unroll, extend the scalars and rebuild the vector.
6453 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
6454 SDValue Val = DAG.getExtractVectorElt(DL, InSVT, InOp, i);
6455 switch (Opcode) {
6457 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
6458 break;
6460 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
6461 break;
6463 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
6464 break;
6465 default:
6466 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6467 }
6468 Ops.push_back(Val);
6469 }
6470
6471 while (Ops.size() != WidenNumElts)
6472 Ops.push_back(DAG.getPOISON(WidenSVT));
6473
6474 return DAG.getBuildVector(WidenVT, DL, Ops);
6475}
6476
6477SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6478 // If this is an FCOPYSIGN with same input types, we can treat it as a
6479 // normal binary op.
6480 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
6481 return WidenVecRes_Binary(N);
6482
6483 // If the types are different, fall back to unrolling.
6484 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6485 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
6486}
6487
6488/// Result and first source operand are different scalar types, but must have
6489/// the same number of elements. There is an additional control argument which
6490/// should be passed through unchanged.
6491SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6492 SDValue FpValue = N->getOperand(0);
6493 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6494 if (getTypeAction(FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6495 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
6496 SDValue Arg = GetWidenedVector(FpValue);
6497 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, {Arg, N->getOperand(1)},
6498 N->getFlags());
6499}
6500
6501SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6502 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6503 SDValue InOp = GetWidenedVector(N->getOperand(0));
6504 SDValue RHS = N->getOperand(1);
6505 EVT ExpVT = RHS.getValueType();
6506 SDValue ExpOp = RHS;
6507 if (ExpVT.isVector()) {
6508 EVT WideExpVT = WidenVT.changeVectorElementType(
6509 *DAG.getContext(), ExpVT.getVectorElementType());
6510 if (WideExpVT == TLI.getTypeToTransformTo(*DAG.getContext(), ExpVT))
6511 ExpOp = GetWidenedVector(RHS);
6512 else
6513 ExpOp = ModifyToType(RHS, WideExpVT);
6514 }
6515
6516 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ExpOp);
6517}
6518
6519SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6520 // Unary op widening.
6521 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6522 SDValue InOp = GetWidenedVector(N->getOperand(0));
6523 if (N->getNumOperands() == 1)
6524 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, N->getFlags());
6525 assert(N->getOpcode() == ISD::AssertNoFPClass && "unexpected opcode");
6526 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, N->getOperand(1),
6527 N->getFlags());
6528}
6529
6530SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6531 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6532 EVT ExtVT = EVT::getVectorVT(
6533 *DAG.getContext(),
6534 cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(),
6535 WidenVT.getVectorElementCount());
6536 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
6537 return DAG.getNode(N->getOpcode(), SDLoc(N),
6538 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
6539}
6540
6541SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6542 unsigned ResNo) {
6543 EVT VT0 = N->getValueType(0);
6544 EVT VT1 = N->getValueType(1);
6545
6546 assert(VT0.isVector() && VT1.isVector() &&
6548 "expected both results to be vectors of matching element count");
6549
6550 LLVMContext &Ctx = *DAG.getContext();
6551 SDValue InOp = GetWidenedVector(N->getOperand(0));
6552
6553 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(ResNo));
6554 ElementCount WidenEC = WidenVT.getVectorElementCount();
6555
6556 EVT WidenVT0 = EVT::getVectorVT(Ctx, VT0.getVectorElementType(), WidenEC);
6557 EVT WidenVT1 = EVT::getVectorVT(Ctx, VT1.getVectorElementType(), WidenEC);
6558
6559 SDNode *WidenNode =
6560 DAG.getNode(N->getOpcode(), SDLoc(N), {WidenVT0, WidenVT1}, InOp)
6561 .getNode();
6562
6563 ReplaceOtherWidenResults(N, WidenNode, ResNo);
6564 return SDValue(WidenNode, ResNo);
6565}
6566
6567SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6568 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6569 return GetWidenedVector(WidenVec);
6570}
6571
6572SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6573 SDLoc DL(N);
6574 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6575 ElementCount WidenEC = WidenVT.getVectorElementCount();
6576 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
6577
6578 // The source has the same number of elements as the result, so widen it to
6579 // match WidenVT. It only lives in the widened-vector map if it is itself
6580 // widened; otherwise pad it up to the widened element count
6581 // when it is illegal.
6582 SDValue InOp = N->getOperand(0);
6583 EVT InVT = InOp.getValueType();
6584 TargetLowering::LegalizeTypeAction InAction = getTypeAction(InVT);
6585 if (InAction == TargetLowering::TypeWidenVector) {
6586 InOp = GetWidenedVector(InOp);
6587 } else if (InAction != TargetLowering::TypeLegal) {
6588 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(),
6589 InVT.getVectorElementType(), WidenEC);
6590 InOp = DAG.getInsertSubvector(DL, DAG.getPOISON(InWidenVT), InOp, 0);
6591 }
6592
6593 return DAG.getAddrSpaceCast(
6594 DL, WidenVT, InOp, AddrSpaceCastN->getSrcAddressSpace(),
6595 AddrSpaceCastN->getDestAddressSpace(), AddrSpaceCastN->getFlags());
6596}
6597
6598SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6599 SDValue InOp = N->getOperand(0);
6600 EVT InVT = InOp.getValueType();
6601 EVT VT = N->getValueType(0);
6602 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6603 SDLoc dl(N);
6604
6605 switch (getTypeAction(InVT)) {
6607 break;
6609 report_fatal_error("scalarization of scalable vectors is not supported");
6611 // If the incoming type is a vector that is being promoted, then
6612 // we know that the elements are arranged differently and that we
6613 // must perform the conversion using a stack slot.
6614 if (InVT.isVector())
6615 break;
6616
6617 // If the InOp is promoted to the same size, convert it. Otherwise,
6618 // fall out of the switch and widen the promoted input.
6619 SDValue NInOp = GetPromotedInteger(InOp);
6620 EVT NInVT = NInOp.getValueType();
6621 if (WidenVT.bitsEq(NInVT)) {
6622 // For big endian targets we need to shift the input integer or the
6623 // interesting bits will end up at the wrong place.
6624 if (DAG.getDataLayout().isBigEndian()) {
6625 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6626 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
6627 DAG.getShiftAmountConstant(ShiftAmt, NInVT, dl));
6628 }
6629 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
6630 }
6631 InOp = NInOp;
6632 InVT = NInVT;
6633 break;
6634 }
6641 break;
6643 // If the InOp is widened to the same size, convert it. Otherwise, fall
6644 // out of the switch and widen the widened input.
6645 InOp = GetWidenedVector(InOp);
6646 InVT = InOp.getValueType();
6647 if (WidenVT.bitsEq(InVT))
6648 // The input widens to the same size. Convert to the widen value.
6649 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
6650 break;
6651 }
6652
6653 unsigned WidenSize = WidenVT.getSizeInBits();
6654 unsigned InSize = InVT.getSizeInBits();
6655 unsigned InScalarSize = InVT.getScalarSizeInBits();
6656 // x86mmx is not an acceptable vector element type, so don't try.
6657 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6658 // Determine new input vector type. The new input vector type will use
6659 // the same element type (if its a vector) or use the input type as a
6660 // vector. It is the same size as the type to widen to.
6661 EVT NewInVT;
6662 unsigned NewNumParts = WidenSize / InSize;
6663 if (InVT.isVector()) {
6664 EVT InEltVT = InVT.getVectorElementType();
6665 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
6666 WidenSize / InEltVT.getSizeInBits());
6667 } else {
6668 // For big endian systems, using the promoted input scalar type
6669 // to produce the scalar_to_vector would put the desired bits into
6670 // the least significant byte(s) of the wider element zero. This
6671 // will mean that the users of the result vector are using incorrect
6672 // bits. Use the original input type instead. Although either input
6673 // type can be used on little endian systems, for consistency we
6674 // use the original type there as well.
6675 EVT OrigInVT = N->getOperand(0).getValueType();
6676 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6677 NewInVT = EVT::getVectorVT(*DAG.getContext(), OrigInVT, NewNumParts);
6678 }
6679
6680 if (TLI.isTypeLegal(NewInVT)) {
6681 SDValue NewVec;
6682 if (InVT.isVector()) {
6683 // Because the result and the input are different vector types, widening
6684 // the result could create a legal type but widening the input might
6685 // make it an illegal type that might lead to repeatedly splitting the
6686 // input and then widening it. To avoid this, we widen the input only if
6687 // it results in a legal type.
6688 if (WidenSize % InSize == 0) {
6689 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(InVT));
6690 Ops[0] = InOp;
6691
6692 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
6693 } else {
6695 DAG.ExtractVectorElements(InOp, Ops);
6696 Ops.append(WidenSize / InScalarSize - Ops.size(),
6697 DAG.getPOISON(InVT.getVectorElementType()));
6698
6699 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops);
6700 }
6701 } else {
6702 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
6703 }
6704 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
6705 }
6706 }
6707
6708 return CreateStackStoreLoad(InOp, WidenVT);
6709}
6710
6711SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6712 return DAG.getNode(
6713 N->getOpcode(), SDLoc(N),
6714 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)),
6715 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3));
6716}
6717
6718SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6719 SDLoc dl(N);
6720 // Build a vector with poison for the new nodes.
6721 EVT VT = N->getValueType(0);
6722
6723 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6724 // type. The POISONs need to have the same type as the existing operands.
6725 EVT EltVT = N->getOperand(0).getValueType();
6726 unsigned NumElts = VT.getVectorNumElements();
6727
6728 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6729 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6730
6731 SmallVector<SDValue, 16> NewOps(N->ops());
6732 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6733 NewOps.append(WidenNumElts - NumElts, DAG.getPOISON(EltVT));
6734
6735 return DAG.getBuildVector(WidenVT, dl, NewOps);
6736}
6737
6738SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6739 EVT InVT = N->getOperand(0).getValueType();
6740 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6741 SDLoc dl(N);
6742 unsigned NumOperands = N->getNumOperands();
6743
6744 bool InputWidened = false; // Indicates we need to widen the input.
6745 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
6746 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6747 unsigned NumInElts = InVT.getVectorMinNumElements();
6748 if (WidenNumElts % NumInElts == 0) {
6749 // Add undef vectors to widen to correct length.
6750 unsigned NumConcat = WidenNumElts / NumInElts;
6751 SDValue UndefVal = DAG.getPOISON(InVT);
6752 SmallVector<SDValue, 16> Ops(NumConcat);
6753 for (unsigned i=0; i < NumOperands; ++i)
6754 Ops[i] = N->getOperand(i);
6755 for (unsigned i = NumOperands; i != NumConcat; ++i)
6756 Ops[i] = UndefVal;
6757 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
6758 }
6759 } else {
6760 InputWidened = true;
6761 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
6762 // The inputs and the result are widen to the same value.
6763 unsigned i;
6764 for (i=1; i < NumOperands; ++i)
6765 if (!N->getOperand(i).isUndef())
6766 break;
6767
6768 if (i == NumOperands)
6769 // Everything but the first operand is an UNDEF so just return the
6770 // widened first operand.
6771 return GetWidenedVector(N->getOperand(0));
6772
6773 if (NumOperands == 2) {
6774 assert(!WidenVT.isScalableVector() &&
6775 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6776 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6777 unsigned NumInElts = InVT.getVectorNumElements();
6778
6779 // Replace concat of two operands with a shuffle.
6780 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6781 for (unsigned i = 0; i < NumInElts; ++i) {
6782 MaskOps[i] = i;
6783 MaskOps[i + NumInElts] = i + WidenNumElts;
6784 }
6785 return DAG.getVectorShuffle(WidenVT, dl,
6786 GetWidenedVector(N->getOperand(0)),
6787 GetWidenedVector(N->getOperand(1)),
6788 MaskOps);
6789 }
6790 }
6791 }
6792
6793 if (WidenVT.isScalableVector()) {
6794 SDValue WideVec = DAG.getPOISON(WidenVT);
6795 unsigned NumInElts = InVT.getVectorMinNumElements();
6796 for (unsigned I = 0; I < NumOperands; ++I)
6797 WideVec =
6798 DAG.getInsertSubvector(dl, WideVec, N->getOperand(I), I * NumInElts);
6799 return WideVec;
6800 }
6801
6802 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6803 unsigned NumInElts = InVT.getVectorNumElements();
6804
6805 // Fall back to use extracts and build vector.
6806 EVT EltVT = WidenVT.getVectorElementType();
6807 SmallVector<SDValue, 16> Ops(WidenNumElts);
6808 unsigned Idx = 0;
6809 for (unsigned i=0; i < NumOperands; ++i) {
6810 SDValue InOp = N->getOperand(i);
6811 if (InputWidened)
6812 InOp = GetWidenedVector(InOp);
6813 for (unsigned j = 0; j < NumInElts; ++j)
6814 Ops[Idx++] = DAG.getExtractVectorElt(dl, EltVT, InOp, j);
6815 }
6816 SDValue UndefVal = DAG.getPOISON(EltVT);
6817 for (; Idx < WidenNumElts; ++Idx)
6818 Ops[Idx] = UndefVal;
6819 return DAG.getBuildVector(WidenVT, dl, Ops);
6820}
6821
6822SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6823 EVT VT = N->getValueType(0);
6824 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6825 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
6826 SDValue InOp2 = N->getOperand(1);
6827 SDValue Idx = N->getOperand(2);
6828 SDLoc dl(N);
6829 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WidenVT, InOp1, InOp2, Idx);
6830}
6831
6832SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6833 EVT VT = N->getValueType(0);
6834 EVT EltVT = VT.getVectorElementType();
6835 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6836 SDValue InOp = N->getOperand(0);
6837 SDValue Idx = N->getOperand(1);
6838 SDLoc dl(N);
6839
6840 auto InOpTypeAction = getTypeAction(InOp.getValueType());
6841 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6842 InOp = GetWidenedVector(InOp);
6843
6844 EVT InVT = InOp.getValueType();
6845
6846 // Check if we can just return the input vector after widening.
6847 uint64_t IdxVal = Idx->getAsZExtVal();
6848 if (IdxVal == 0 && InVT == WidenVT)
6849 return InOp;
6850
6851 // Check if we can extract from the vector.
6852 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6853 unsigned InNumElts = InVT.getVectorMinNumElements();
6854 unsigned VTNumElts = VT.getVectorMinNumElements();
6855 assert(IdxVal % VTNumElts == 0 &&
6856 "Expected Idx to be a multiple of subvector minimum vector length");
6857 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6858 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
6859
6860 if (VT.isScalableVector()) {
6861 // Try to split the operation up into smaller extracts and concat the
6862 // results together, e.g.
6863 // nxv6i64 extract_subvector(nxv12i64, 6)
6864 // <->
6865 // nxv8i64 concat(
6866 // nxv2i64 extract_subvector(nxv16i64, 6)
6867 // nxv2i64 extract_subvector(nxv16i64, 8)
6868 // nxv2i64 extract_subvector(nxv16i64, 10)
6869 // undef)
6870 unsigned GCD = std::gcd(VTNumElts, WidenNumElts);
6871 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6872 "down type's element count");
6873 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
6875 // Avoid recursion around e.g. nxv1i8.
6876 if (getTypeAction(PartVT) != TargetLowering::TypeWidenVector) {
6878 unsigned I = 0;
6879 for (; I < VTNumElts / GCD; ++I)
6880 Parts.push_back(
6881 DAG.getExtractSubvector(dl, PartVT, InOp, IdxVal + I * GCD));
6882 for (; I < WidenNumElts / GCD; ++I)
6883 Parts.push_back(DAG.getPOISON(PartVT));
6884
6885 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts);
6886 }
6887
6888 // Fallback to extracting through memory.
6889
6890 Align Alignment = DAG.getReducedAlign(InVT, /*UseABI=*/false);
6891 SDValue StackPtr = DAG.CreateStackTemporary(InVT.getStoreSize(), Alignment);
6892 MachineFunction &MF = DAG.getMachineFunction();
6893 int FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
6894 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
6895
6896 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6899 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6902
6903 // Write out the input vector.
6904 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, StoreMMO);
6905
6906 // Build a mask to match the length of the non-widened result.
6907 SDValue Mask =
6908 DAG.getMaskFromElementCount(dl, WidenVT, VT.getVectorElementCount());
6909
6910 // Read back the sub-vector setting the remaining lanes to poison.
6911 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, InVT, VT, Idx);
6912 return DAG.getMaskedLoad(
6913 WidenVT, dl, Ch, StackPtr, DAG.getPOISON(StackPtr.getValueType()), Mask,
6914 DAG.getPOISON(WidenVT), VT, LoadMMO, ISD::UNINDEXED, ISD::NON_EXTLOAD);
6915 }
6916
6917 // We could try widening the input to the right length but for now, extract
6918 // the original elements, fill the rest with undefs and build a vector.
6919 SmallVector<SDValue, 16> Ops(WidenNumElts);
6920 unsigned i;
6921 for (i = 0; i < VTNumElts; ++i)
6922 Ops[i] = DAG.getExtractVectorElt(dl, EltVT, InOp, IdxVal + i);
6923
6924 SDValue UndefVal = DAG.getPOISON(EltVT);
6925 for (; i < WidenNumElts; ++i)
6926 Ops[i] = UndefVal;
6927 return DAG.getBuildVector(WidenVT, dl, Ops);
6928}
6929
6930SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6931 SDValue InOp = ModifyToType(
6932 N->getOperand(0),
6933 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), true);
6934 return DAG.getNode(ISD::AssertZext, SDLoc(N), InOp.getValueType(), InOp,
6935 N->getOperand(1));
6936}
6937
6938SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6939 SDValue InOp = GetWidenedVector(N->getOperand(0));
6940 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
6941 InOp.getValueType(), InOp,
6942 N->getOperand(1), N->getOperand(2));
6943}
6944
6945/// Either return the same load or provide appropriate casts
6946/// from the load and return that.
6947static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6948 TypeSize LdWidth, TypeSize FirstVTWidth,
6949 SDLoc dl, SelectionDAG &DAG) {
6950 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6951 "Load width must be less than or equal to first value type width");
6952 TypeSize WidenWidth = WidenVT.getSizeInBits();
6953 if (!FirstVT.isVector()) {
6954 unsigned NumElts =
6955 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6956 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), FirstVT, NumElts);
6957 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
6958 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
6959 }
6960 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6961 return LdOp;
6962}
6963
6964/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6965/// widened value so it can be issued in a single atomic store.
6966static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6967 TypeSize FirstVTWidth, const SDLoc &dl,
6968 SelectionDAG &DAG) {
6969 TypeSize WidenWidth = WidenVT.getSizeInBits();
6970 if (!FirstVT.isVector()) {
6971 unsigned NumElts =
6972 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6973 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), FirstVT, NumElts);
6974 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, StVal);
6975 return DAG.getExtractVectorElt(dl, FirstVT, VecOp, 0);
6976 }
6977 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6978 return StVal;
6979}
6980
6981static std::optional<EVT> findMemType(SelectionDAG &DAG,
6982 const TargetLowering &TLI, unsigned Width,
6983 EVT WidenVT, unsigned Align,
6984 unsigned WidenEx);
6985
6986SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6987 EVT WidenVT =
6988 TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
6989 EVT LdVT = LD->getMemoryVT();
6990 SDLoc dl(LD);
6991
6992 // Load information
6993 SDValue Chain = LD->getChain();
6994 SDValue BasePtr = LD->getBasePtr();
6995
6996 TypeSize LdWidth = LdVT.getSizeInBits();
6997 TypeSize WidenWidth = WidenVT.getSizeInBits();
6998 TypeSize WidthDiff = WidenWidth - LdWidth;
6999
7000 // Find the vector type that can load from.
7001 std::optional<EVT> FirstVT =
7002 findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/0,
7003 WidthDiff.getKnownMinValue());
7004
7005 if (!FirstVT)
7006 return SDValue();
7007
7008 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
7009
7010 SDValue LdOp = DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, *FirstVT, *FirstVT,
7011 Chain, BasePtr, LD->getMemOperand());
7012
7013 // Load the element with one instruction.
7014 SDValue Result = coerceLoadedValue(LdOp, *FirstVT, WidenVT, LdWidth,
7015 FirstVTWidth, dl, DAG);
7016
7017 // Modified the chain - switch anything that used the old chain to use
7018 // the new one.
7019 ReplaceValueWith(SDValue(LD, 1), LdOp.getValue(1));
7020 return Result;
7021}
7022
7023SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
7024 LoadSDNode *LD = cast<LoadSDNode>(N);
7025 ISD::LoadExtType ExtType = LD->getExtensionType();
7026
7027 // A vector must always be stored in memory as-is, i.e. without any padding
7028 // between the elements, since various code depend on it, e.g. in the
7029 // handling of a bitcast of a vector type to int, which may be done with a
7030 // vector store followed by an integer load. A vector that does not have
7031 // elements that are byte-sized must therefore be stored as an integer
7032 // built out of the extracted vector elements.
7033 if (!LD->getMemoryVT().isByteSized()) {
7034 SDValue Value, NewChain;
7035 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
7036 ReplaceValueWith(SDValue(LD, 0), Value);
7037 ReplaceValueWith(SDValue(LD, 1), NewChain);
7038 return SDValue();
7039 }
7040
7041 // Generate a vector-predicated load if it is custom/legal on the target. To
7042 // avoid possible recursion, only do this if the widened mask type is legal.
7043 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
7044 // removed from the IR by the ExpandVectorPredication pass but we're
7045 // reintroducing them here.
7046 EVT VT = LD->getValueType(0);
7047 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7048 EVT WideMaskVT = getSetCCResultType(WideVT);
7049
7050 if (ExtType == ISD::NON_EXTLOAD &&
7051 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WideVT) &&
7052 TLI.isTypeLegal(WideMaskVT)) {
7053 SDLoc DL(N);
7054 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT);
7055 SDValue EVL = DAG.getElementCount(DL, TLI.getVPExplicitVectorLengthTy(),
7057 SDValue NewLoad =
7058 DAG.getLoadVP(LD->getAddressingMode(), ISD::NON_EXTLOAD, WideVT, DL,
7059 LD->getChain(), LD->getBasePtr(), LD->getOffset(), Mask,
7060 EVL, LD->getMemoryVT(), LD->getMemOperand());
7061
7062 // Modified the chain - switch anything that used the old chain to use
7063 // the new one.
7064 ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1));
7065
7066 return NewLoad;
7067 }
7068
7070 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
7071 if (ExtType != ISD::NON_EXTLOAD)
7072 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
7073 else
7074 Result = GenWidenVectorLoads(LdChain, LD);
7075
7076 if (Result) {
7077 // If we generate a single load, we can use that for the chain. Otherwise,
7078 // build a factor node to remember the multiple loads are independent and
7079 // chain to that.
7080 SDValue NewChain;
7081 if (LdChain.size() == 1)
7082 NewChain = LdChain[0];
7083 else
7084 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
7085
7086 // Modified the chain - switch anything that used the old chain to use
7087 // the new one.
7088 ReplaceValueWith(SDValue(N, 1), NewChain);
7089
7090 return Result;
7091 }
7092
7093 if (VT.isVector()) {
7094 // If all else fails replace the load with a wide masked load.
7095 SDLoc DL(N);
7096 SDValue Mask =
7097 DAG.getMaskFromElementCount(DL, WideVT, VT.getVectorElementCount());
7098
7099 SDValue NewLoad = DAG.getMaskedLoad(
7100 WideVT, DL, LD->getChain(), LD->getBasePtr(), LD->getOffset(), Mask,
7101 DAG.getPOISON(WideVT), LD->getMemoryVT(), LD->getMemOperand(),
7102 LD->getAddressingMode(), LD->getExtensionType());
7103
7104 ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1));
7105 return NewLoad;
7106 }
7107
7108 report_fatal_error("Unable to widen vector load");
7109}
7110
7111SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
7112 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7113 SDValue Mask = N->getMask();
7114 SDValue EVL = N->getVectorLength();
7115 ISD::LoadExtType ExtType = N->getExtensionType();
7116 SDLoc dl(N);
7117
7118 // The mask should be widened as well
7119 assert(getTypeAction(Mask.getValueType()) ==
7121 "Unable to widen binary VP op");
7122 Mask = GetWidenedVector(Mask);
7123 assert(Mask.getValueType().getVectorElementCount() ==
7124 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
7125 .getVectorElementCount() &&
7126 "Unable to widen vector load");
7127
7128 SDValue Res =
7129 DAG.getLoadVP(N->getAddressingMode(), ExtType, WidenVT, dl, N->getChain(),
7130 N->getBasePtr(), N->getOffset(), Mask, EVL,
7131 N->getMemoryVT(), N->getMemOperand(), N->isExpandingLoad());
7132 // Legalize the chain result - switch anything that used the old chain to
7133 // use the new one.
7134 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
7135 return Res;
7136}
7137
7138SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
7139 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7140 SDValue Mask = N->getMask();
7141 SDValue EVL = N->getVectorLength();
7142 SDLoc dl(N);
7143
7144 // The mask should be widened as well
7145 assert(getTypeAction(Mask.getValueType()) ==
7147 "Unable to widen binary VP op");
7148 Mask = GetWidenedVector(Mask);
7149 assert(Mask.getValueType().getVectorElementCount() ==
7150 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
7151 .getVectorElementCount() &&
7152 "Unable to widen vector load");
7153
7154 SDValue Res = DAG.getLoadFFVP(WidenVT, dl, N->getChain(), N->getBasePtr(),
7155 Mask, EVL, N->getMemOperand());
7156 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
7157 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
7158 return Res;
7159}
7160
7161SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
7162 SDLoc DL(N);
7163
7164 // The mask should be widened as well
7165 SDValue Mask = N->getMask();
7166 assert(getTypeAction(Mask.getValueType()) ==
7168 "Unable to widen VP strided load");
7169 Mask = GetWidenedVector(Mask);
7170
7171 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7172 assert(Mask.getValueType().getVectorElementCount() ==
7173 WidenVT.getVectorElementCount() &&
7174 "Data and mask vectors should have the same number of elements");
7175
7176 SDValue Res = DAG.getStridedLoadVP(
7177 N->getAddressingMode(), N->getExtensionType(), WidenVT, DL, N->getChain(),
7178 N->getBasePtr(), N->getOffset(), N->getStride(), Mask,
7179 N->getVectorLength(), N->getMemoryVT(), N->getMemOperand(),
7180 N->isExpandingLoad());
7181
7182 // Legalize the chain result - switch anything that used the old chain to
7183 // use the new one.
7184 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
7185 return Res;
7186}
7187
7188SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
7189 SDValue Vec = N->getOperand(0);
7190 SDValue Mask = N->getOperand(1);
7191 SDValue Passthru = N->getOperand(2);
7192 EVT WideVecVT =
7193 TLI.getTypeToTransformTo(*DAG.getContext(), Vec.getValueType());
7194 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
7195 Mask.getValueType().getVectorElementType(),
7196 WideVecVT.getVectorElementCount());
7197
7198 SDValue WideVec = ModifyToType(Vec, WideVecVT);
7199 SDValue WideMask = ModifyToType(Mask, WideMaskVT, /*FillWithZeroes=*/true);
7200 SDValue WidePassthru = ModifyToType(Passthru, WideVecVT);
7201 return DAG.getNode(ISD::VECTOR_COMPRESS, SDLoc(N), WideVecVT, WideVec,
7202 WideMask, WidePassthru);
7203}
7204
7205SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
7206 // Only indices that were out of range can select the new source elements.
7207 SDValue Src = GetWidenedVector(N->getOperand(0));
7208 EVT WideVT = Src.getValueType();
7209 EVT WideMaskVT = WideVT.changeVectorElementType(
7210 *DAG.getContext(),
7211 N->getOperand(1).getValueType().getVectorElementType());
7212 SDValue Mask = ModifyToType(N->getOperand(1), WideMaskVT);
7213 return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, SDLoc(N), WideVT, Src, Mask);
7214}
7215
7216SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
7217 EVT VT = N->getValueType(0);
7218 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7219 SDValue Mask = N->getMask();
7220 EVT MaskVT = Mask.getValueType();
7221 SDValue PassThru = GetWidenedVector(N->getPassThru());
7222 ISD::LoadExtType ExtType = N->getExtensionType();
7223 SDLoc dl(N);
7224
7225 EVT WideMaskVT =
7226 EVT::getVectorVT(*DAG.getContext(), MaskVT.getVectorElementType(),
7227 WidenVT.getVectorElementCount());
7228
7229 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
7230 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WidenVT) &&
7231 TLI.isTypeLegal(WideMaskVT) &&
7232 // If there is a passthru, we shouldn't use vp.load. However,
7233 // type legalizer will struggle on masked.load with
7234 // scalable vectors, so for scalable vectors, we still use vp.load
7235 // but manually merge the load result with the passthru using vp.select.
7236 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
7237 Mask = DAG.getInsertSubvector(dl, DAG.getPOISON(WideMaskVT), Mask, 0);
7238 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
7240 SDValue NewLoad =
7241 DAG.getLoadVP(N->getAddressingMode(), ISD::NON_EXTLOAD, WidenVT, dl,
7242 N->getChain(), N->getBasePtr(), N->getOffset(), Mask, EVL,
7243 N->getMemoryVT(), N->getMemOperand());
7244 SDValue NewVal = NewLoad;
7245
7246 // Manually merge with vselect
7247 if (!N->getPassThru()->isUndef()) {
7248 assert(WidenVT.isScalableVector());
7249 NewVal = DAG.getNode(ISD::VSELECT, dl, WidenVT, Mask, NewVal, PassThru);
7250 // The lanes past EVL are poison.
7251 NewVal = DAG.getNode(ISD::VP_MERGE, dl, WidenVT,
7252 DAG.getAllOnesConstant(dl, WideMaskVT), NewVal,
7253 DAG.getPOISON(WidenVT), EVL);
7254 }
7255
7256 // Modified the chain - switch anything that used the old chain to use
7257 // the new one.
7258 ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1));
7259
7260 return NewVal;
7261 }
7262
7263 // The mask should be widened as well
7264 Mask = ModifyToType(Mask, WideMaskVT, true);
7265
7266 SDValue Res = DAG.getMaskedLoad(
7267 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
7268 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
7269 ExtType, N->isExpandingLoad());
7270 // Legalize the chain result - switch anything that used the old chain to
7271 // use the new one.
7272 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
7273 return Res;
7274}
7275
7276SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
7277
7278 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7279 SDValue Mask = N->getMask();
7280 EVT MaskVT = Mask.getValueType();
7281 SDValue PassThru = GetWidenedVector(N->getPassThru());
7282 SDValue Scale = N->getScale();
7283 ElementCount WideEC = WideVT.getVectorElementCount();
7284 SDLoc dl(N);
7285
7286 // The mask should be widened as well
7287 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
7288 MaskVT.getVectorElementType(), WideEC);
7289 Mask = ModifyToType(Mask, WideMaskVT, true);
7290
7291 // Widen the Index operand
7292 SDValue Index = N->getIndex();
7293 EVT WideIndexVT = EVT::getVectorVT(
7294 *DAG.getContext(), Index.getValueType().getScalarType(), WideEC);
7295 Index = ModifyToType(Index, WideIndexVT);
7296 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7297 Scale };
7298
7299 // Widen the MemoryType
7300 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
7301 N->getMemoryVT().getScalarType(), WideEC);
7302 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
7303 WideMemVT, dl, Ops, N->getMemOperand(),
7304 N->getIndexType(), N->getExtensionType());
7305
7306 // Legalize the chain result - switch anything that used the old chain to
7307 // use the new one.
7308 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
7309 return Res;
7310}
7311
7312SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7313 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7314 SDValue Mask = N->getMask();
7315 SDValue Scale = N->getScale();
7316 ElementCount WideEC = WideVT.getVectorElementCount();
7317 SDLoc dl(N);
7318
7319 SDValue Index = GetWidenedVector(N->getIndex());
7320 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
7321 N->getMemoryVT().getScalarType(), WideEC);
7322 Mask = GetWidenedMask(Mask, WideEC);
7323
7324 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7325 Mask, N->getVectorLength()};
7326 SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT,
7327 dl, Ops, N->getMemOperand(), N->getIndexType());
7328
7329 // Legalize the chain result - switch anything that used the old chain to
7330 // use the new one.
7331 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
7332 return Res;
7333}
7334
7335SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7336 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7337 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
7338}
7339
7340// Return true is this is a SETCC node or a strict version of it.
7341static inline bool isSETCCOp(unsigned Opcode) {
7342 switch (Opcode) {
7343 case ISD::SETCC:
7344 case ISD::STRICT_FSETCC:
7346 return true;
7347 }
7348 return false;
7349}
7350
7351// Return true if this is a node that could have two SETCCs as operands.
7352static inline bool isLogicalMaskOp(unsigned Opcode) {
7353 switch (Opcode) {
7354 case ISD::AND:
7355 case ISD::OR:
7356 case ISD::XOR:
7357 return true;
7358 }
7359 return false;
7360}
7361
7362// If N is a SETCC or a strict variant of it, return the type
7363// of the compare operands.
7365 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7366 return N->getOperand(OpNo).getValueType();
7367}
7368
7369// This is used just for the assert in convertMask(). Check that this either
7370// a SETCC or a previously handled SETCC by convertMask().
7371#ifndef NDEBUG
7372static inline bool isSETCCorConvertedSETCC(SDValue N) {
7373 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7374 N = N.getOperand(0);
7375 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7376 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7377 if (!N->getOperand(i)->isUndef())
7378 return false;
7379 N = N.getOperand(0);
7380 }
7381
7382 if (N.getOpcode() == ISD::TRUNCATE)
7383 N = N.getOperand(0);
7384 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7385 N = N.getOperand(0);
7386
7387 if (isLogicalMaskOp(N.getOpcode()))
7388 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7389 isSETCCorConvertedSETCC(N.getOperand(1));
7390
7391 return (isSETCCOp(N.getOpcode()) || ISD::isBuildVectorOfConstantSDNodes(N));
7392}
7393#endif
7394
7395// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7396// to ToMaskVT if needed with vector extension or truncation.
7397SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7398 EVT ToMaskVT) {
7399 // Called from convertMaskTree for SETCC leaf nodes. Re-creates the SETCC with
7400 // result type MaskVT, then sign-extends/truncates and pads to ToMaskVT.
7401 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7402
7403 // Make a new Mask node, with a legal result VT.
7404 SDValue Mask;
7406 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7407 Ops.push_back(InMask->getOperand(i));
7408 if (InMask->isStrictFPOpcode()) {
7409 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
7410 { MaskVT, MVT::Other }, Ops);
7411 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
7412 }
7413 else
7414 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops,
7415 InMask->getFlags());
7416
7417 return adjustMaskToType(Mask, ToMaskVT);
7418}
7419
7420// Adjust element width (sign-extend/truncate) and element count
7421// (extract/concat) of Mask to match ToMaskVT.
7422SDValue DAGTypeLegalizer::adjustMaskToType(SDValue Mask, EVT ToMaskVT) {
7423 LLVMContext &Ctx = *DAG.getContext();
7424 EVT MaskVT = Mask.getValueType();
7425 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7426 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7427 if (MaskScalarBits < ToMaskScalBits) {
7428 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
7429 MaskVT.getVectorNumElements());
7430 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
7431 } else if (MaskScalarBits > ToMaskScalBits) {
7432 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
7433 MaskVT.getVectorNumElements());
7434 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
7435 }
7436
7437 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7438 ToMaskVT.getScalarSizeInBits() &&
7439 "Mask should have the right element size by now.");
7440
7441 // Adjust Mask to the right number of elements.
7442 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
7443 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7444 Mask = DAG.getExtractSubvector(SDLoc(Mask), ToMaskVT, Mask, 0);
7445 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7446 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7447 EVT SubVT = Mask->getValueType(0);
7448 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(SubVT));
7449 SubOps[0] = Mask;
7450 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
7451 }
7452
7453 assert((Mask->getValueType(0) == ToMaskVT) &&
7454 "A mask of ToMaskVT should have been produced by now.");
7455
7456 return Mask;
7457}
7458
7459// Adjust both operands to a common intermediate mask type, picking a scalar
7460// width that minimizes extend/truncate overhead given the final target ToVT.
7461EVT DAGTypeLegalizer::unifyMaskTypes(SDValue &Op0, bool IsOpLenient0,
7462 SDValue &Op1, bool IsOpLenient1,
7463 EVT ToVT) {
7466 "unifyMaskTypes only handles scalar width differences");
7467
7468 // If only one of the operands lenient type-wise, we can simply
7469 // adjust its type to the other operand's type assuming that this
7470 // adjustment can be folded away.
7471 //
7472 // NOTE: We essentially rely on the fact that further optimizations
7473 // do spot redundant casts in "lenient" cases. If at some
7474 // point we decide that we want to do better here, we can
7475 // postpone converting lenient sub-trees right away and postpone
7476 // it to the moment when we know the best fitting integer type
7477 // to materialize them, and do it there.
7478 if (IsOpLenient0 != IsOpLenient1) {
7479 SDValue *LenientOp, *NonLenientOp;
7480 if (IsOpLenient0) {
7481 LenientOp = &Op0;
7482 NonLenientOp = &Op1;
7483 } else {
7484 LenientOp = &Op1;
7485 NonLenientOp = &Op0;
7486 }
7487 EVT OpVT = NonLenientOp->getValueType();
7488 *LenientOp = adjustMaskToType(*LenientOp, OpVT);
7489 return OpVT;
7490 }
7491
7492 unsigned Bits0 = Op0.getScalarValueSizeInBits();
7493 unsigned Bits1 = Op1.getScalarValueSizeInBits();
7494 unsigned NarrowBits = std::min(Bits0, Bits1);
7495 unsigned WideBits = std::max(Bits0, Bits1);
7496 unsigned ToBits = ToVT.getScalarSizeInBits();
7497 unsigned IntBits = NarrowBits == WideBits ? NarrowBits
7498 : ToBits >= WideBits ? WideBits
7499 : ToBits <= NarrowBits ? NarrowBits
7500 : ToBits;
7501 EVT OpVT = Op0.getValueType().changeVectorElementType(
7502 *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), IntBits));
7503 Op0 = adjustMaskToType(Op0, OpVT);
7504 Op1 = adjustMaskToType(Op1, OpVT);
7505 return OpVT;
7506}
7507
7508std::pair<SDValue, bool>
7509DAGTypeLegalizer::convertMaskTreeImpl(SDValue V, EVT ToVT, unsigned Depth) {
7510 // The main idea is to recursively traverse VSELECT's mask that needs
7511 // widening to see if we can avoid unnecessary casts. The problem usually
7512 // stems from a simple fact that SETCC might naturally produce results
7513 // not in i1 (as we model it in LLVM IR) and we can continue using that
7514 // type until we have to switch it up. Another important aspect is that
7515 // "all ones" and "all zeros" constants can be materialized at any type,
7516 // so we can try to utilize that to keep SETCC results at their natural
7517 // types as much as possible.
7518 //
7519 // The algorithm traverses the mask-producing tree of operations that
7520 // retain "mask-vector"-ness of the input (i.e. it remains a vector of
7521 // -1s and 0s).
7522 //
7523 // SETCC1 SETCC2 CONST1 SETCC3 CONST2 CONST3
7524 // | / | / | /
7525 // | / | / | /
7526 // |_____/ |______/ |______/
7527 // | * choose the | * choose SETCC3 | * keep it as final type
7528 // | most fitting | type | but consider it subject to
7529 // | type | / change
7530 // | | /
7531 // | |______________/
7532 // | | * choose SETCC3 type
7533 // | /
7534 // | /
7535 // |_____________/
7536 // | * choose the most fitting type
7537 // | and then cast to the final desired type
7538 // |
7539 // VSELECT
7540 //
7541 if (Depth >= DAG.MaxRecursionDepth)
7542 return {};
7543
7544 // Bail out when encounter the vector element count mismatch.
7545 // It potentially can be just an assertion, but we deliberately try to
7546 // be overly conservative here.
7547 if (V.getValueType().getVectorNumElements() != ToVT.getVectorNumElements())
7548 return {};
7549
7550 unsigned Opcode = V.getOpcode();
7551
7552 // Base case: SETCC produces the mask at its natural type.
7553 if (isSETCCOp(Opcode)) {
7554 EVT MaskVT = getSetCCResultType(getSETCCOperandType(V));
7555 return {convertMask(V, MaskVT, MaskVT), /*IsTypeLenient=*/false};
7556 }
7557
7558 SDLoc DL(V);
7559
7560 // Base case: all-zeros or all-ones BUILD_VECTOR. Type-lenient since these are
7561 // invariant under sign-extend/truncate.
7563 return {DAG.getConstant(0, DL, ToVT), /*IsTypeLenient=*/true};
7565 return {DAG.getAllOnesConstant(DL, ToVT), /*IsTypeLenient=*/true};
7566
7567 // Logical operations (AND/OR/XOR): try picking the best fitting width out
7568 // of children's element widths.
7569 if (isLogicalMaskOp(Opcode)) {
7570 auto [Op0, IsLenientOp0] =
7571 convertMaskTreeImpl(V.getOperand(0), ToVT, Depth + 1);
7572 if (!Op0)
7573 return {};
7574 auto [Op1, IsLenientOp1] =
7575 convertMaskTreeImpl(V.getOperand(1), ToVT, Depth + 1);
7576 if (!Op1)
7577 return {};
7578 EVT OpVT = unifyMaskTypes(Op0, IsLenientOp0, Op1, IsLenientOp1, ToVT);
7579 return {DAG.getNode(Opcode, DL, OpVT, Op0, Op1),
7580 IsLenientOp0 && IsLenientOp1};
7581 }
7582
7583 // FREEZE: widen the operand and re-wrap.
7584 if (Opcode == ISD::FREEZE) {
7585 auto [Inner, IsTypeLenient] =
7586 convertMaskTreeImpl(V.getOperand(0), ToVT, Depth + 1);
7587 if (!Inner)
7588 return {};
7589 return {DAG.getNode(ISD::FREEZE, DL, Inner.getValueType(), Inner),
7590 IsTypeLenient};
7591 }
7592
7593 // Vector shuffle: try inferring the best fitting width from operands.
7594 if (Opcode == ISD::VECTOR_SHUFFLE) {
7595 auto *Shuf = cast<ShuffleVectorSDNode>(V);
7596 auto [Op0, IsLenientOp0] =
7597 convertMaskTreeImpl(V.getOperand(0), ToVT, Depth + 1);
7598 if (!Op0)
7599 return {};
7600 if (V.getOperand(1).isUndef()) {
7601 EVT OpVT = Op0.getValueType();
7602 return {DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT),
7603 Shuf->getMask()),
7604 IsLenientOp0};
7605 }
7606 auto [Op1, IsLenientOp1] =
7607 convertMaskTreeImpl(V.getOperand(1), ToVT, Depth + 1);
7608 if (!Op1)
7609 return {};
7610 EVT OpVT = unifyMaskTypes(Op0, IsLenientOp0, Op1, IsLenientOp1, ToVT);
7611 return {DAG.getVectorShuffle(OpVT, DL, Op0, Op1, Shuf->getMask()),
7612 IsLenientOp0 && IsLenientOp1};
7613 }
7614
7615 // SELECT/VSELECT: try inferring the best fitting width from operands.
7616 if (Opcode == ISD::SELECT || Opcode == ISD::VSELECT) {
7617 auto [Op1, IsLenientOp1] =
7618 convertMaskTreeImpl(V.getOperand(1), ToVT, Depth + 1);
7619 if (!Op1)
7620 return {};
7621 auto [Op2, IsLenientOp2] =
7622 convertMaskTreeImpl(V.getOperand(2), ToVT, Depth + 1);
7623 if (!Op2)
7624 return {};
7625 EVT OpVT = unifyMaskTypes(Op1, IsLenientOp1, Op2, IsLenientOp2, ToVT);
7626
7627 // We deliberately skip traversing/modifying VSELECT's mask because
7628 //
7629 // a. We only change bitwidth of the operands and it shouldn't affect
7630 // condition on its own.
7631 //
7632 // b. This VSELECT's mask can be widened in an independent traversal
7633 // if needed.
7634 SDValue Cond = V.getOperand(0);
7635 return {DAG.getNode(Opcode, DL, OpVT, Cond, Op1, Op2),
7636 IsLenientOp1 && IsLenientOp2};
7637 }
7638
7639 return {};
7640}
7641
7642SDValue DAGTypeLegalizer::convertMaskTree(SDValue V, EVT ToVT) {
7643 // In general, we are converting from <N x i1> into <M x iW>.
7644 // This would mean that during the tree traversal we need to pay
7645 // attention to both bitwidth and element count, which can be error-prone.
7646 //
7647 // Instead, we split the task in two, we first widen the type of the tree
7648 // and then change the element count.
7649 EVT MaskTreeVT = ToVT.changeVectorElementCount(
7650 *DAG.getContext(), V.getValueType().getVectorElementCount());
7651 auto [Result, _] = convertMaskTreeImpl(V, MaskTreeVT);
7652 if (!Result)
7653 return Result;
7654 return adjustMaskToType(Result, ToVT);
7655}
7656
7657// This method tries to handle some special cases for the vselect mask
7658// and if needed adjusting the mask vector type to match that of the VSELECT.
7659// Without it, many cases end up with scalarization of the SETCC, with many
7660// unnecessary instructions.
7661SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7662 LLVMContext &Ctx = *DAG.getContext();
7663 SDValue Cond = N->getOperand(0);
7664
7665 if (N->getOpcode() != ISD::VSELECT)
7666 return SDValue();
7667
7668 // If this is a splitted VSELECT that was previously already handled, do
7669 // nothing.
7670 EVT CondVT = Cond->getValueType(0);
7671 if (CondVT.getScalarSizeInBits() != 1)
7672 return SDValue();
7673
7674 EVT VSelVT = N->getValueType(0);
7675
7676 // This method can't handle scalable vector types.
7677 // FIXME: This support could be added in the future.
7678 if (VSelVT.isScalableVector())
7679 return SDValue();
7680
7681 // Only handle vector types which are a power of 2.
7682 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
7683 return SDValue();
7684
7685 // Don't touch if this will be scalarized.
7686 EVT FinalVT = VSelVT;
7687 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
7688 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
7689
7690 if (FinalVT.getVectorNumElements() == 1)
7691 return SDValue();
7692
7693 // If there is support for an i1 vector mask, don't touch.
7694 if (isSETCCOp(Cond.getOpcode())) {
7695 EVT SetCCOpVT = getSETCCOperandType(Cond);
7696 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
7697 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
7698 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
7699 if (SetCCResVT.getScalarSizeInBits() == 1)
7700 return SDValue();
7701 } else if (CondVT.getScalarType() == MVT::i1) {
7702 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7703 // don't touch.
7704 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
7705 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
7706
7707 if (CondVT.getScalarType() == MVT::i1)
7708 return SDValue();
7709 }
7710
7711 // Widen the vselect result type if needed.
7712 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
7713 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
7714
7715 // The mask of the VSELECT should have integer elements.
7716 EVT ToMaskVT = VSelVT;
7717 if (!ToMaskVT.getScalarType().isInteger())
7718 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7719
7720 // Try to recursively widen the mask expression tree to the target type.
7721 return convertMaskTree(Cond, ToMaskVT);
7722}
7723
7724SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7725 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7726 ElementCount WidenEC = WidenVT.getVectorElementCount();
7727
7728 SDValue Cond1 = N->getOperand(0);
7729 EVT CondVT = Cond1.getValueType();
7730 unsigned Opcode = N->getOpcode();
7731 if (CondVT.isVector()) {
7732 if (SDValue WideCond = WidenVSELECTMask(N)) {
7733 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
7734 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
7735 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7736 return DAG.getNode(Opcode, SDLoc(N), WidenVT, WideCond, InOp1, InOp2);
7737 }
7738
7739 EVT CondEltVT = CondVT.getVectorElementType();
7740 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenEC);
7741 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
7742 Cond1 = GetWidenedVector(Cond1);
7743
7744 // If we have to split the condition there is no point in widening the
7745 // select. This would result in an cycle of widening the select ->
7746 // widening the condition operand -> splitting the condition operand ->
7747 // splitting the select -> widening the select. Instead split this select
7748 // further and widen the resulting type.
7749 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
7750 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
7751 SDValue Res = ModifyToType(SplitSelect, WidenVT);
7752 return Res;
7753 }
7754
7755 if (Cond1.getValueType() != CondWidenVT)
7756 Cond1 = ModifyToType(Cond1, CondWidenVT);
7757 }
7758
7759 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
7760 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
7761 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7762 if (Opcode == ISD::VP_MERGE)
7763 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2,
7764 N->getOperand(3));
7765 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2);
7766}
7767
7768SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7769 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
7770 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
7771 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
7772 InOp1.getValueType(), N->getOperand(0),
7773 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
7774}
7775
7776SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7777 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7778 return DAG.getUNDEF(WidenVT);
7779}
7780
7781SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7782 EVT VT = N->getValueType(0);
7783 SDLoc dl(N);
7784
7785 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7786 unsigned NumElts = VT.getVectorNumElements();
7787 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7788
7789 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
7790 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
7791
7792 // Adjust mask based on new input vector length.
7793 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7794 for (unsigned i = 0; i != NumElts; ++i) {
7795 int Idx = N->getMaskElt(i);
7796 if (Idx < (int)NumElts)
7797 NewMask[i] = Idx;
7798 else
7799 NewMask[i] = Idx - NumElts + WidenNumElts;
7800 }
7801 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
7802}
7803
7804SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7805 EVT VT = N->getValueType(0);
7806 EVT EltVT = VT.getVectorElementType();
7807 SDLoc dl(N);
7808
7809 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7810 SDValue OpValue = GetWidenedVector(N->getOperand(0));
7811 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7812
7813 SDValue ReverseVal = DAG.getNode(ISD::VECTOR_REVERSE, dl, WidenVT, OpValue);
7814 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7815 unsigned VTNumElts = VT.getVectorMinNumElements();
7816 unsigned IdxVal = WidenNumElts - VTNumElts;
7817
7818 if (VT.isScalableVector()) {
7819 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7820 // results together, e.g.(nxv6i64 -> nxv8i64)
7821 // nxv8i64 vector_reverse
7822 // <->
7823 // nxv8i64 concat(
7824 // nxv2i64 extract_subvector(nxv8i64, 2)
7825 // nxv2i64 extract_subvector(nxv8i64, 4)
7826 // nxv2i64 extract_subvector(nxv8i64, 6)
7827 // nxv2i64 undef)
7828
7829 unsigned GCD = std::gcd(VTNumElts, WidenNumElts);
7830 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
7832 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7833 "down type's element count");
7835 unsigned i = 0;
7836 for (; i < VTNumElts / GCD; ++i)
7837 Parts.push_back(
7838 DAG.getExtractSubvector(dl, PartVT, ReverseVal, IdxVal + i * GCD));
7839 for (; i < WidenNumElts / GCD; ++i)
7840 Parts.push_back(DAG.getPOISON(PartVT));
7841
7842 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts);
7843 }
7844
7845 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7846 // fixed-vectors.
7847 SmallVector<int, 16> Mask(WidenNumElts, -1);
7848 std::iota(Mask.begin(), Mask.begin() + VTNumElts, IdxVal);
7849
7850 return DAG.getVectorShuffle(WidenVT, dl, ReverseVal, DAG.getPOISON(WidenVT),
7851 Mask);
7852}
7853
7854SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7855 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7856 return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
7857}
7858
7859void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
7860 EVT VT = N->getValueType(0);
7861 EVT EltVT = VT.getVectorElementType();
7862 ElementCount OrigEC = VT.getVectorElementCount();
7863 unsigned Factor = N->getNumOperands();
7864 SDLoc DL(N);
7865
7866 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7867 ElementCount WidenEC = WidenVT.getVectorElementCount();
7868
7869 SmallVector<SDValue, 8> WidenOps(Factor);
7870 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7871 WidenOps[Idx] = GetWidenedVector(N->getOperand(Idx));
7872
7873 SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
7874 SDValue Interleaved =
7875 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, WidenVTs, WidenOps);
7876
7877 EVT PackedWidenVT =
7878 EVT::getVectorVT(*DAG.getContext(), EltVT, WidenEC * Factor);
7879 SmallVector<SDValue, 8> Slices(Factor);
7880 for (unsigned Idx = 0; Idx != Factor; ++Idx)
7881 Slices[Idx] = Interleaved.getValue(Idx);
7882
7883 SDValue Packed = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedWidenVT, Slices);
7884
7885 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7886 SDValue Narrow = DAG.getExtractSubvector(DL, VT, Packed,
7887 OrigEC.getKnownMinValue() * Idx);
7888 SDValue Wide =
7889 DAG.getInsertSubvector(DL, DAG.getPOISON(WidenVT), Narrow, /*Idx=*/0U);
7890 SetWidenedVector(SDValue(N, Idx), Wide);
7891 }
7892}
7893
7894SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
7895 SDLoc DL(N);
7896 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7897 EVT SourceVT = N->getOperand(0).getValueType();
7898 EVT WideSourceVT =
7899 EVT::getVectorVT(*DAG.getContext(), SourceVT.getVectorElementType(),
7900 WidenVT.getVectorElementCount());
7901
7902 SDValue WideSource = DAG.getInsertSubvector(DL, DAG.getUNDEF(WideSourceVT),
7903 N->getOperand(0), 0);
7904 SDValue WideMask = DAG.getInsertSubvector(DL, DAG.getConstant(0, DL, WidenVT),
7905 N->getOperand(2), 0);
7906 return DAG.getNode(ISD::VECTOR_MATCH, DL, WidenVT, WideSource,
7907 N->getOperand(1), WideMask, N->getFlags());
7908}
7909
7910void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7911 EVT VT = N->getValueType(0);
7912 EVT EltVT = VT.getVectorElementType();
7913 ElementCount OrigEC = VT.getVectorElementCount();
7914 unsigned Factor = N->getNumOperands();
7915 SDLoc DL(N);
7916
7917 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7918 ElementCount WidenEC = WidenVT.getVectorElementCount();
7919 // We cannot just use the widened operands directly: since they might be
7920 // individually widened, using them directly will result in de-interleaving
7921 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7922 // not concat the widened operands but the original ones to effectively
7923 // generate a "packed" concated and widened vector, before extracting new
7924 // operand vectors with the widened type.
7925 EVT PackedWidenVT =
7926 EVT::getVectorVT(*DAG.getContext(), EltVT, WidenEC * Factor);
7927 EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), EltVT, OrigEC * Factor);
7928 SDValue ConcatOp = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, N->ops());
7929 SDValue PackedWidenVec = DAG.getInsertSubvector(
7930 DL, DAG.getUNDEF(PackedWidenVT), ConcatOp, /*Idx=*/0U);
7931
7932 // Extract the new widened operand vectors.
7933 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7934 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7935 NewOps[Idx] = DAG.getExtractSubvector(DL, WidenVT, PackedWidenVec,
7936 WidenEC.getKnownMinValue() * Idx);
7937 }
7938
7939 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7940 SDValue NewRes = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, NewVTs, NewOps);
7941 // Set the widened results manually.
7942 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7943 SetWidenedVector(SDValue(N, Idx), NewRes.getValue(Idx));
7944}
7945
7946SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7947 assert(N->getValueType(0).isVector() &&
7948 N->getOperand(0).getValueType().isVector() &&
7949 "Operands must be vectors");
7950 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
7951 ElementCount WidenEC = WidenVT.getVectorElementCount();
7952
7953 SDValue InOp1 = N->getOperand(0);
7954 EVT InVT = InOp1.getValueType();
7955 assert(InVT.isVector() && "can not widen non-vector type");
7956 EVT WidenInVT =
7957 EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenEC);
7958
7959 // The input and output types often differ here, and it could be that while
7960 // we'd prefer to widen the result type, the input operands have been split.
7961 // In this case, we also need to split the result of this node as well.
7962 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
7963 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7964 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
7965 return Res;
7966 }
7967
7968 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7969 SDValue InOp2 = N->getOperand(1);
7970 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
7971 InOp1 = GetWidenedVector(InOp1);
7972 InOp2 = GetWidenedVector(InOp2);
7973 } else {
7974 SDValue Poison = DAG.getPOISON(WidenInVT);
7975 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(N));
7976 InOp1 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WidenInVT, Poison,
7977 InOp1, ZeroIdx);
7978 InOp2 = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), WidenInVT, Poison,
7979 InOp2, ZeroIdx);
7980 }
7981
7982 // Assume that the input and output will be widen appropriately. If not,
7983 // we will have to unroll it at some point.
7984 assert(InOp1.getValueType() == WidenInVT &&
7985 InOp2.getValueType() == WidenInVT &&
7986 "Input not widened to expected type!");
7987 (void)WidenInVT;
7988 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, InOp1, InOp2,
7989 N->getOperand(2));
7990}
7991
7992SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7993 assert(N->getValueType(0).isVector() &&
7994 N->getOperand(1).getValueType().isVector() &&
7995 "Operands must be vectors");
7996 EVT VT = N->getValueType(0);
7997 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7998 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7999 unsigned NumElts = VT.getVectorNumElements();
8000 EVT EltVT = VT.getVectorElementType();
8001
8002 SDLoc dl(N);
8003 SDValue Chain = N->getOperand(0);
8004 SDValue LHS = N->getOperand(1);
8005 SDValue RHS = N->getOperand(2);
8006 SDValue CC = N->getOperand(3);
8007 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8008
8009 // Fully unroll and reassemble.
8010 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(EltVT));
8011 SmallVector<SDValue, 8> Chains(NumElts);
8012 for (unsigned i = 0; i != NumElts; ++i) {
8013 SDValue LHSElem = DAG.getExtractVectorElt(dl, TmpEltVT, LHS, i);
8014 SDValue RHSElem = DAG.getExtractVectorElt(dl, TmpEltVT, RHS, i);
8015
8016 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
8017 {Chain, LHSElem, RHSElem, CC});
8018 Chains[i] = Scalars[i].getValue(1);
8019 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
8020 DAG.getBoolConstant(true, dl, EltVT, VT),
8021 DAG.getBoolConstant(false, dl, EltVT, VT));
8022 }
8023
8024 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
8025 ReplaceValueWith(SDValue(N, 1), NewChain);
8026
8027 return DAG.getBuildVector(WidenVT, dl, Scalars);
8028}
8029
8030SDValue DAGTypeLegalizer::WidenVecRes_PARTIAL_REDUCE_MLA(SDNode *N) {
8031 SDLoc DL(N);
8032 EVT VT = N->getValueType(0);
8033
8034 // Expand, then widen the result.
8035 SDValue Expanded = TLI.expandPartialReduceMLA(N, DAG);
8036 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
8037 return DAG.getInsertSubvector(DL, DAG.getPOISON(WideVT), Expanded, 0);
8038}
8039
8040//===----------------------------------------------------------------------===//
8041// Widen Vector Operand
8042//===----------------------------------------------------------------------===//
8043bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
8044 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
8045 SDValue Res = SDValue();
8046
8047 // See if the target wants to custom widen this node.
8048 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
8049 return false;
8050
8051 switch (N->getOpcode()) {
8052 default:
8053#ifndef NDEBUG
8054 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
8055 N->dump(&DAG);
8056 dbgs() << "\n";
8057#endif
8058 report_fatal_error("do not know how to widen this operator's operand!");
8059
8060 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
8061 case ISD::FAKE_USE:
8062 Res = WidenVecOp_FAKE_USE(N);
8063 break;
8064 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
8065 case ISD::VECTOR_REPEAT:
8066 Res = WidenVecOp_VECTOR_REPEAT(N);
8067 break;
8068 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
8069 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
8070 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
8071 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
8072 case ISD::ATOMIC_STORE:
8073 Res = WidenVecOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
8074 break;
8075 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
8076 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8077 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
8078 break;
8082 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
8083 break;
8084 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
8085 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
8086 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
8087 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
8088 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
8089 case ISD::STRICT_FSETCC:
8090 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
8091 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
8092 case ISD::FLDEXP:
8093 case ISD::FCOPYSIGN:
8094 case ISD::LROUND:
8095 case ISD::LLROUND:
8096 case ISD::LRINT:
8097 case ISD::LLRINT:
8098 Res = WidenVecOp_UnrollVectorOp(N);
8099 break;
8100 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
8101
8102 case ISD::ANY_EXTEND:
8103 case ISD::SIGN_EXTEND:
8104 case ISD::ZERO_EXTEND:
8105 Res = WidenVecOp_EXTEND(N);
8106 break;
8107
8108 case ISD::SCMP:
8109 case ISD::UCMP:
8110 Res = WidenVecOp_CMP(N);
8111 break;
8112
8113 case ISD::FP_EXTEND:
8115 case ISD::FP_ROUND:
8117 case ISD::FP_TO_SINT:
8119 case ISD::FP_TO_UINT:
8121 case ISD::SINT_TO_FP:
8123 case ISD::UINT_TO_FP:
8125 case ISD::TRUNCATE:
8128 Res = WidenVecOp_Convert(N);
8129 break;
8130
8133 Res = WidenVecOp_FP_TO_XINT_SAT(N);
8134 break;
8135
8138 case ISD::VECREDUCE_ADD:
8139 case ISD::VECREDUCE_MUL:
8140 case ISD::VECREDUCE_AND:
8141 case ISD::VECREDUCE_OR:
8142 case ISD::VECREDUCE_XOR:
8153 Res = WidenVecOp_VECREDUCE(N);
8154 break;
8157 Res = WidenVecOp_VECREDUCE_SEQ(N);
8158 break;
8159 case ISD::VP_REDUCE_FADD:
8160 case ISD::VP_REDUCE_SEQ_FADD:
8161 case ISD::VP_REDUCE_FMUL:
8162 case ISD::VP_REDUCE_SEQ_FMUL:
8163 case ISD::VP_REDUCE_ADD:
8164 case ISD::VP_REDUCE_MUL:
8165 case ISD::VP_REDUCE_AND:
8166 case ISD::VP_REDUCE_OR:
8167 case ISD::VP_REDUCE_XOR:
8168 case ISD::VP_REDUCE_SMAX:
8169 case ISD::VP_REDUCE_SMIN:
8170 case ISD::VP_REDUCE_UMAX:
8171 case ISD::VP_REDUCE_UMIN:
8172 case ISD::VP_REDUCE_FMAX:
8173 case ISD::VP_REDUCE_FMIN:
8174 case ISD::VP_REDUCE_FMAXIMUM:
8175 case ISD::VP_REDUCE_FMINIMUM:
8176 Res = WidenVecOp_VP_REDUCE(N);
8177 break;
8178 case ISD::CTTZ_ELTS:
8180 Res = WidenVecOp_CttzElements(N);
8181 break;
8182 case ISD::VP_CTTZ_ELTS:
8183 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8184 Res = WidenVecOp_VP_CttzElements(N);
8185 break;
8187 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
8188 break;
8190 Res = WidenVecOp_VECTOR_SHUFFLE_VAR(N, OpNo);
8191 break;
8192 case ISD::VECTOR_MATCH:
8193 Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
8194 break;
8199 Res = WidenVecOp_PARTIAL_REDUCE_MLA(N);
8200 break;
8201 }
8202
8203 // If Res is null, the sub-method took care of registering the result.
8204 if (!Res.getNode()) return false;
8205
8206 // If the result is N, the sub-method updated N in place. Tell the legalizer
8207 // core about this.
8208 if (Res.getNode() == N)
8209 return true;
8210
8211
8212 if (N->isStrictFPOpcode())
8213 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
8214 "Invalid operand expansion");
8215 else
8216 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
8217 "Invalid operand expansion");
8218
8219 ReplaceValueWith(SDValue(N, 0), Res);
8220 return false;
8221}
8222
8223SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
8224 SDLoc DL(N);
8225 EVT VT = N->getValueType(0);
8226
8227 SDValue InOp = N->getOperand(0);
8228 assert(getTypeAction(InOp.getValueType()) ==
8230 "Unexpected type action");
8231 InOp = GetWidenedVector(InOp);
8234 "Input wasn't widened!");
8235
8236 // We may need to further widen the operand until it has the same total
8237 // vector size as the result.
8238 EVT InVT = InOp.getValueType();
8239 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
8240 EVT InEltVT = InVT.getVectorElementType();
8241 for (EVT FixedVT : MVT::vector_valuetypes()) {
8242 EVT FixedEltVT = FixedVT.getVectorElementType();
8243 if (TLI.isTypeLegal(FixedVT) &&
8244 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
8245 FixedEltVT == InEltVT) {
8246 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
8247 "Not enough elements in the fixed type for the operand!");
8248 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
8249 "We can't have the same type as we started with!");
8250 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
8251 InOp = DAG.getInsertSubvector(DL, DAG.getPOISON(FixedVT), InOp, 0);
8252 else
8253 InOp = DAG.getExtractSubvector(DL, FixedVT, InOp, 0);
8254 break;
8255 }
8256 }
8257 InVT = InOp.getValueType();
8258 if (InVT.getSizeInBits() != VT.getSizeInBits())
8259 // We couldn't find a legal vector type that was a widening of the input
8260 // and could be extended in-register to the result type, so we have to
8261 // scalarize.
8262 return WidenVecOp_Convert(N);
8263 }
8264
8265 // Use special DAG nodes to represent the operation of extending the
8266 // low lanes.
8267 switch (N->getOpcode()) {
8268 default:
8269 llvm_unreachable("Extend legalization on extend operation!");
8270 case ISD::ANY_EXTEND:
8271 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
8272 case ISD::SIGN_EXTEND:
8273 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
8274 case ISD::ZERO_EXTEND:
8275 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
8276 }
8277}
8278
8279SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
8280 SDLoc dl(N);
8281
8282 EVT OpVT = N->getOperand(0).getValueType();
8283 EVT ResVT = N->getValueType(0);
8284 SDValue LHS = GetWidenedVector(N->getOperand(0));
8285 SDValue RHS = GetWidenedVector(N->getOperand(1));
8286
8287 // 1. EXTRACT_SUBVECTOR
8288 // 2. SIGN_EXTEND/ZERO_EXTEND
8289 // 3. CMP
8290 LHS = DAG.getExtractSubvector(dl, OpVT, LHS, 0);
8291 RHS = DAG.getExtractSubvector(dl, OpVT, RHS, 0);
8292
8293 // At this point the result type is guaranteed to be valid, so we can use it
8294 // as the operand type by extending it appropriately
8295 ISD::NodeType ExtendOpcode =
8296 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8297 LHS = DAG.getNode(ExtendOpcode, dl, ResVT, LHS);
8298 RHS = DAG.getNode(ExtendOpcode, dl, ResVT, RHS);
8299
8300 return DAG.getNode(N->getOpcode(), dl, ResVT, LHS, RHS);
8301}
8302
8303SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
8304 // The result (and first input) is legal, but the second input is illegal.
8305 // We can't do much to fix that, so just unroll and let the extracts off of
8306 // the second input be widened as needed later.
8307 return DAG.UnrollVectorOp(N);
8308}
8309
8310SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
8311 SDLoc DL(N);
8312 EVT ResultVT = N->getValueType(0);
8313 SDValue Test = N->getOperand(1);
8314 SDValue WideArg = GetWidenedVector(N->getOperand(0));
8315
8316 // Process this node similarly to SETCC.
8317 EVT WideResultVT = getSetCCResultType(WideArg.getValueType());
8318 if (ResultVT.getScalarType() == MVT::i1)
8319 WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
8320 WideResultVT.getVectorNumElements());
8321
8322 SDValue WideNode = DAG.getNode(ISD::IS_FPCLASS, DL, WideResultVT,
8323 {WideArg, Test}, N->getFlags());
8324
8325 // Extract the needed results from the result vector.
8326 EVT ResVT =
8327 EVT::getVectorVT(*DAG.getContext(), WideResultVT.getVectorElementType(),
8328 ResultVT.getVectorNumElements());
8329 SDValue CC = DAG.getExtractSubvector(DL, ResVT, WideNode, 0);
8330
8331 EVT OpVT = N->getOperand(0).getValueType();
8332 ISD::NodeType ExtendCode =
8333 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
8334 return DAG.getNode(ExtendCode, DL, ResultVT, CC);
8335}
8336
8337SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
8338 // Since the result is legal and the input is illegal.
8339 EVT VT = N->getValueType(0);
8340 EVT EltVT = VT.getVectorElementType();
8341 SDLoc dl(N);
8342 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
8343 assert(getTypeAction(InOp.getValueType()) ==
8345 "Unexpected type action");
8346 InOp = GetWidenedVector(InOp);
8347 EVT InVT = InOp.getValueType();
8348 unsigned Opcode = N->getOpcode();
8349
8350 // Helper to build a convert node with all scalar trailing operands.
8351 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
8352 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
8353 return DAG.getNode(Opcode, dl, VT, Op, N->getOperand(1), N->getOperand(2),
8354 N->getOperand(3));
8355 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
8356 return DAG.getNode(Opcode, dl, VT, Op, N->getOperand(1));
8357 return DAG.getNode(Opcode, dl, VT, Op);
8358 };
8359
8360 // See if a widened result type would be legal, if so widen the node.
8361 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
8362 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
8363 InVT.getVectorElementCount());
8364 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
8365 SDValue Res;
8366 if (N->isStrictFPOpcode()) {
8367 if (Opcode == ISD::STRICT_FP_ROUND)
8368 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
8369 { N->getOperand(0), InOp, N->getOperand(2) });
8370 else
8371 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
8372 { N->getOperand(0), InOp });
8373 // Legalize the chain result - switch anything that used the old chain to
8374 // use the new one.
8375 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
8376 } else {
8377 Res = MakeConvertNode(WideVT, InOp);
8378 }
8379 return DAG.getExtractSubvector(dl, VT, Res, 0);
8380 }
8381
8382 EVT InEltVT = InVT.getVectorElementType();
8383
8384 // Unroll the convert into some scalar code and create a nasty build vector.
8385 unsigned NumElts = VT.getVectorNumElements();
8387 if (N->isStrictFPOpcode()) {
8388 SmallVector<SDValue, 4> NewOps(N->ops());
8389 SmallVector<SDValue, 32> OpChains;
8390 for (unsigned i=0; i < NumElts; ++i) {
8391 NewOps[1] = DAG.getExtractVectorElt(dl, InEltVT, InOp, i);
8392 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
8393 OpChains.push_back(Ops[i].getValue(1));
8394 }
8395 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
8396 ReplaceValueWith(SDValue(N, 1), NewChain);
8397 } else {
8398 for (unsigned i = 0; i < NumElts; ++i) {
8399 SDValue Elt = DAG.getExtractVectorElt(dl, InEltVT, InOp, i);
8400 Ops[i] = MakeConvertNode(EltVT, Elt);
8401 }
8402 }
8403
8404 return DAG.getBuildVector(VT, dl, Ops);
8405}
8406
8407SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
8408 EVT DstVT = N->getValueType(0);
8409 SDValue Src = GetWidenedVector(N->getOperand(0));
8410 EVT SrcVT = Src.getValueType();
8411 ElementCount WideNumElts = SrcVT.getVectorElementCount();
8412 SDLoc dl(N);
8413
8414 // See if a widened result type would be legal, if so widen the node.
8415 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(),
8416 DstVT.getVectorElementType(), WideNumElts);
8417 if (TLI.isTypeLegal(WideDstVT)) {
8418 SDValue Res =
8419 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1));
8420 return DAG.getNode(
8421 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res,
8422 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
8423 }
8424
8425 // Give up and unroll.
8426 return DAG.UnrollVectorOp(N);
8427}
8428
8429SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
8430 EVT VT = N->getValueType(0);
8431 SDValue InOp = GetWidenedVector(N->getOperand(0));
8432 EVT InWidenVT = InOp.getValueType();
8433 SDLoc dl(N);
8434
8435 // Check if we can convert between two legal vector types and extract.
8436 TypeSize InWidenSize = InWidenVT.getSizeInBits();
8437 TypeSize Size = VT.getSizeInBits();
8438 // x86mmx is not an acceptable vector element type, so don't try.
8439 if (!VT.isVector() && VT != MVT::x86mmx &&
8440 InWidenSize.hasKnownScalarFactor(Size)) {
8441 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(Size);
8442 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
8443 if (TLI.isTypeLegal(NewVT)) {
8444 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
8445 return DAG.getExtractVectorElt(dl, VT, BitOp, 0);
8446 }
8447 }
8448
8449 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
8450 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
8451 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
8452 // having to copy via memory.
8453 if (VT.isVector()) {
8454 EVT EltVT = VT.getVectorElementType();
8455 unsigned EltSize = EltVT.getFixedSizeInBits();
8456 if (InWidenSize.isKnownMultipleOf(EltSize)) {
8457 ElementCount NewNumElts =
8458 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
8459 .divideCoefficientBy(EltSize);
8460 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
8461 if (TLI.isTypeLegal(NewVT)) {
8462 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
8463 return DAG.getExtractSubvector(dl, VT, BitOp, 0);
8464 }
8465 }
8466 }
8467
8468 return CreateStackStoreLoad(InOp, VT);
8469}
8470
8471// Vectors with sizes that are not powers of 2 need to be widened to the
8472// next largest power of 2. For example, we may get a vector of 3 32-bit
8473// integers or of 6 16-bit integers, both of which have to be widened to a
8474// 128-bit vector.
8475SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8476 SDValue WidenedOp = GetWidenedVector(N->getOperand(1));
8477 return DAG.getNode(ISD::FAKE_USE, SDLoc(), MVT::Other, N->getOperand(0),
8478 WidenedOp);
8479}
8480
8481SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8482 EVT VT = N->getValueType(0);
8483 EVT EltVT = VT.getVectorElementType();
8484 EVT InVT = N->getOperand(0).getValueType();
8485 SDLoc dl(N);
8486
8487 // If the widen width for this operand is the same as the width of the concat
8488 // and all but the first operand is undef, just use the widened operand.
8489 unsigned NumOperands = N->getNumOperands();
8490 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
8491 unsigned i;
8492 for (i = 1; i < NumOperands; ++i)
8493 if (!N->getOperand(i).isUndef())
8494 break;
8495
8496 if (i == NumOperands)
8497 return GetWidenedVector(N->getOperand(0));
8498 }
8499
8500 if (VT.isScalableVector()) {
8501 SDValue Result = DAG.getPOISON(VT);
8502 unsigned NumInElts = InVT.getVectorMinNumElements();
8503 for (unsigned i = 0; i < NumOperands; ++i) {
8504 SDValue InOp = GetWidenedVector(N->getOperand(i));
8505 if (InOp.getValueType() != InVT)
8506 InOp = DAG.getExtractSubvector(dl, InVT, InOp, 0);
8507 Result = DAG.getInsertSubvector(dl, Result, InOp, i * NumInElts);
8508 }
8509 return Result;
8510 }
8511
8512 // Otherwise, fall back to a nasty build vector.
8513 unsigned NumElts = VT.getVectorNumElements();
8515
8516 unsigned NumInElts = InVT.getVectorNumElements();
8517
8518 unsigned Idx = 0;
8519 for (unsigned i=0; i < NumOperands; ++i) {
8520 SDValue InOp = N->getOperand(i);
8521 assert(getTypeAction(InOp.getValueType()) ==
8523 "Unexpected type action");
8524 InOp = GetWidenedVector(InOp);
8525 for (unsigned j = 0; j < NumInElts; ++j)
8526 Ops[Idx++] = DAG.getExtractVectorElt(dl, EltVT, InOp, j);
8527 }
8528 return DAG.getBuildVector(VT, dl, Ops);
8529}
8530
8531SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_REPEAT(SDNode *N) {
8532 SDLoc DL(N);
8533 EVT VT = N->getValueType(0);
8534 SDValue Src = N->getOperand(0);
8535 EVT SrcVT = Src.getValueType();
8536 EVT WidenedSrcVT = TLI.getTypeToTransformTo(*DAG.getContext(), SrcVT);
8537
8538 if (!WidenedSrcVT.getVectorElementCount().hasKnownScalarFactor(
8539 SrcVT.getVectorElementCount()))
8541 "Cannot widen VECTOR_REPEAT operand to an ElementCount that's not "
8542 "a known scalar multiple of the input ElementCount.");
8543
8544 // Repeat the original source because the extra lanes of its widened value
8545 // are unspecified.
8546 unsigned NumConcat =
8547 WidenedSrcVT.getVectorNumElements() / SrcVT.getVectorNumElements();
8548 SmallVector<SDValue, 8> Ops(NumConcat, Src);
8549 SDValue WidenedSrc = DAG.getNode(ISD::CONCAT_VECTORS, DL, WidenedSrcVT, Ops);
8550 EVT WidenedVT = VT.changeVectorElementCount(
8551 *DAG.getContext(),
8553 SDValue Widened = DAG.getNode(ISD::VECTOR_REPEAT, DL, WidenedVT, WidenedSrc);
8554 return DAG.getExtractSubvector(DL, VT, Widened, 0);
8555}
8556
8557SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8558 EVT VT = N->getValueType(0);
8559 SDValue SubVec = N->getOperand(1);
8560 SDValue InVec = N->getOperand(0);
8561
8562 EVT OrigVT = SubVec.getValueType();
8563 SubVec = GetWidenedVector(SubVec);
8564 EVT SubVT = SubVec.getValueType();
8565
8566 // Whether or not all the elements of the widened SubVec will be inserted into
8567 // valid indices of VT.
8568 bool IndicesValid = false;
8569 // If we statically know that VT can fit SubVT, the indices are valid.
8570 if (VT.knownBitsGE(SubVT))
8571 IndicesValid = true;
8572 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8573 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8574 // we know the minimum vscale we can work out if it's valid ourselves.
8575 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8576 Attribute::VScaleRange);
8577 if (Attr.isValid()) {
8578 unsigned VScaleMin = Attr.getVScaleRangeMin();
8579 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8580 SubVT.getFixedSizeInBits())
8581 IndicesValid = true;
8582 }
8583 }
8584
8585 if (!IndicesValid)
8587 "Don't know how to widen the operands for INSERT_SUBVECTOR");
8588
8589 SDLoc DL(N);
8590
8591 // We need to make sure that the indices are still valid, otherwise we might
8592 // widen what was previously well-defined to something undefined.
8593 if (InVec.isUndef() && N->getConstantOperandVal(2) == 0)
8594 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, InVec, SubVec,
8595 N->getOperand(2));
8596
8597 if (OrigVT.isScalableVector()) {
8598 // When the widened types match, overwriting the start of a vector is
8599 // effectively a merge operation that can be implement as a vselect.
8600 if (SubVT == VT && N->getConstantOperandVal(2) == 0) {
8601 SDValue Mask =
8602 DAG.getMaskFromElementCount(DL, VT, OrigVT.getVectorElementCount());
8603 return DAG.getNode(ISD::VSELECT, DL, VT, Mask, SubVec, InVec);
8604 }
8605
8606 // Fallback to inserting through memory.
8607 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8608 SDValue StackPtr = DAG.CreateStackTemporary(VT.getStoreSize(), Alignment);
8609 MachineFunction &MF = DAG.getMachineFunction();
8610 int FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
8611 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
8612
8613 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8616 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8619
8620 // Write out the vector being inserting into.
8621 SDValue Ch =
8622 DAG.getStore(DAG.getEntryNode(), DL, InVec, StackPtr, StoreMMO);
8623
8624 // Build a mask to match the length of the sub-vector.
8625 SDValue Mask =
8626 DAG.getMaskFromElementCount(DL, SubVT, OrigVT.getVectorElementCount());
8627
8628 // Overwrite the sub-vector at the required offset.
8629 SDValue SubVecPtr =
8630 TLI.getVectorSubVecPointer(DAG, StackPtr, VT, OrigVT, N->getOperand(2));
8631 Ch = DAG.getMaskedStore(Ch, DL, SubVec, SubVecPtr,
8632 DAG.getPOISON(SubVecPtr.getValueType()), Mask, VT,
8633 StoreMMO, ISD::UNINDEXED, ISD::NON_EXTLOAD);
8634
8635 // Read back the result.
8636 return DAG.getLoad(VT, DL, Ch, StackPtr, LoadMMO);
8637 }
8638
8639 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8640 // with a series of INSERT_VECTOR_ELT
8641 unsigned Idx = N->getConstantOperandVal(2);
8642
8643 SDValue InsertElt = InVec;
8644 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8645 SDValue ExtractElt =
8646 DAG.getExtractVectorElt(DL, VT.getVectorElementType(), SubVec, I);
8647 InsertElt = DAG.getInsertVectorElt(DL, InsertElt, ExtractElt, I + Idx);
8648 }
8649
8650 return InsertElt;
8651}
8652
8653SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8654 SDValue InOp = GetWidenedVector(N->getOperand(0));
8655 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
8656 N->getValueType(0), InOp, N->getOperand(1));
8657}
8658
8659SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8660 SDValue InOp = GetWidenedVector(N->getOperand(0));
8661 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
8662 N->getValueType(0), InOp, N->getOperand(1));
8663}
8664
8665SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8666 SDLoc DL(N);
8667 EVT ResVT = N->getValueType(0);
8668
8669 // Widen the input as requested by the legalizer.
8670 SDValue WideInOp = GetWidenedVector(N->getOperand(0));
8671 EVT WideInVT = WideInOp.getValueType();
8672
8673 // Simple case: if widened input is still smaller than or equal to result,
8674 // just use it directly.
8675 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8676 return DAG.getNode(N->getOpcode(), DL, ResVT, WideInOp);
8677
8678 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8679 // If widening makes the input larger than the original result, widen the
8680 // result to match, then extract back down.
8681 EVT ResEltVT = ResVT.getVectorElementType();
8682 unsigned EltBits = ResEltVT.getSizeInBits();
8683 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8684 "Widened input size must be a multiple of result element size");
8685
8686 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8687 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), ResEltVT, WideNumElts);
8688
8689 SDValue WideRes = DAG.getNode(N->getOpcode(), DL, WideResVT, WideInOp);
8690 return DAG.getExtractSubvector(DL, ResVT, WideRes, 0);
8691}
8692
8693SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8694 // We have to widen the value, but we want only to store the original
8695 // vector type.
8696 StoreSDNode *ST = cast<StoreSDNode>(N);
8697
8698 if (!ST->getMemoryVT().getScalarType().isByteSized())
8699 return TLI.scalarizeVectorStore(ST, DAG);
8700
8701 if (ST->isTruncatingStore())
8702 return TLI.scalarizeVectorStore(ST, DAG);
8703
8704 // Generate a vector-predicated store if it is custom/legal on the target.
8705 // To avoid possible recursion, only do this if the widened mask type is
8706 // legal.
8707 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8708 // removed from the IR by the ExpandVectorPredication pass but we're
8709 // reintroducing them here.
8710 SDValue StVal = ST->getValue();
8711 EVT StVT = StVal.getValueType();
8712 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StVT);
8713 EVT WideMaskVT = getSetCCResultType(WideVT);
8714
8715 if (TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) &&
8716 TLI.isTypeLegal(WideMaskVT)) {
8717 // Widen the value.
8718 SDLoc DL(N);
8719 StVal = GetWidenedVector(StVal);
8720 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT);
8721 SDValue EVL = DAG.getElementCount(DL, TLI.getVPExplicitVectorLengthTy(),
8722 StVT.getVectorElementCount());
8723 return DAG.getStoreVP(ST->getChain(), DL, StVal, ST->getBasePtr(),
8724 ST->getOffset(), Mask, EVL, StVT, ST->getMemOperand(),
8725 ST->getAddressingMode());
8726 }
8727
8729 if (GenWidenVectorStores(StChain, ST)) {
8730 if (StChain.size() == 1)
8731 return StChain[0];
8732
8733 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
8734 }
8735
8736 if (StVT.isVector()) {
8737 // If all else fails replace the store with a wide masked store.
8738 SDLoc DL(N);
8739 SDValue WideStVal = GetWidenedVector(StVal);
8740 SDValue Mask =
8741 DAG.getMaskFromElementCount(DL, WideVT, StVT.getVectorElementCount());
8742
8743 return DAG.getMaskedStore(ST->getChain(), DL, WideStVal, ST->getBasePtr(),
8744 ST->getOffset(), Mask, ST->getMemoryVT(),
8745 ST->getMemOperand(), ST->getAddressingMode(),
8746 ST->isTruncatingStore());
8747 }
8748
8749 report_fatal_error("Unable to widen vector store");
8750}
8751
8752SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8753 EVT StVT = ST->getMemoryVT();
8754 SDLoc dl(ST);
8755
8756 SDValue StVal = GetWidenedVector(ST->getVal());
8757 EVT WidenVT = StVal.getValueType();
8758
8759 TypeSize StWidth = StVT.getSizeInBits();
8760 TypeSize WidenWidth = WidenVT.getSizeInBits();
8761 TypeSize WidthDiff = WidenWidth - StWidth;
8762
8763 // Find the vector type that can store the original memory width in one
8764 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8765 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8766 // implies a 4-byte movl, writing undef bytes past its object).
8767 std::optional<EVT> FirstVT =
8768 findMemType(DAG, TLI, StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/0,
8769 WidthDiff.getKnownMinValue());
8770 if (!FirstVT)
8771 return SDValue();
8772
8773 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8774
8775 SDValue StOp =
8776 coerceStoredValue(StVal, *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8777
8778 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, *FirstVT, ST->getChain(), StOp,
8779 ST->getBasePtr(), ST->getMemOperand());
8780}
8781
8782SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8783 assert((OpNo == 1 || OpNo == 3) &&
8784 "Can widen only data or mask operand of vp_store");
8785 VPStoreSDNode *ST = cast<VPStoreSDNode>(N);
8786 SDValue Mask = ST->getMask();
8787 SDValue StVal = ST->getValue();
8788 SDLoc dl(N);
8789
8790 if (OpNo == 1) {
8791 // Widen the value.
8792 StVal = GetWidenedVector(StVal);
8793
8794 // We only handle the case where the mask needs widening to an
8795 // identically-sized type as the vector inputs.
8796 assert(getTypeAction(Mask.getValueType()) ==
8798 "Unable to widen VP store");
8799 Mask = GetWidenedVector(Mask);
8800 } else {
8801 Mask = GetWidenedVector(Mask);
8802
8803 // We only handle the case where the stored value needs widening to an
8804 // identically-sized type as the mask.
8805 assert(getTypeAction(StVal.getValueType()) ==
8807 "Unable to widen VP store");
8808 StVal = GetWidenedVector(StVal);
8809 }
8810
8811 assert(Mask.getValueType().getVectorElementCount() ==
8813 "Mask and data vectors should have the same number of elements");
8814 return DAG.getStoreVP(ST->getChain(), dl, StVal, ST->getBasePtr(),
8815 ST->getOffset(), Mask, ST->getVectorLength(),
8816 ST->getMemoryVT(), ST->getMemOperand(),
8817 ST->getAddressingMode(), ST->isTruncatingStore(),
8818 ST->isCompressingStore());
8819}
8820
8821SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8822 unsigned OpNo) {
8823 assert((OpNo == 1 || OpNo == 4) &&
8824 "Can widen only data or mask operand of vp_strided_store");
8825 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);
8826 SDValue Mask = SST->getMask();
8827 SDValue StVal = SST->getValue();
8828 SDLoc DL(N);
8829
8830 if (OpNo == 1)
8831 assert(getTypeAction(Mask.getValueType()) ==
8833 "Unable to widen VP strided store");
8834 else
8835 assert(getTypeAction(StVal.getValueType()) ==
8837 "Unable to widen VP strided store");
8838
8839 StVal = GetWidenedVector(StVal);
8840 Mask = GetWidenedVector(Mask);
8841
8843 Mask.getValueType().getVectorElementCount() &&
8844 "Data and mask vectors should have the same number of elements");
8845
8846 return DAG.getStridedStoreVP(
8847 SST->getChain(), DL, StVal, SST->getBasePtr(), SST->getOffset(),
8848 SST->getStride(), Mask, SST->getVectorLength(), SST->getMemoryVT(),
8849 SST->getMemOperand(), SST->getAddressingMode(), SST->isTruncatingStore(),
8850 SST->isCompressingStore());
8851}
8852
8853SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8854 assert((OpNo == 1 || OpNo == 4) &&
8855 "Can widen only data or mask operand of mstore");
8856 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
8857 SDValue Mask = MST->getMask();
8858 EVT MaskVT = Mask.getValueType();
8859 SDValue StVal = MST->getValue();
8860 EVT VT = StVal.getValueType();
8861 SDLoc dl(N);
8862
8863 EVT WideVT, WideMaskVT;
8864 if (OpNo == 1) {
8865 // Widen the value.
8866 StVal = GetWidenedVector(StVal);
8867
8868 WideVT = StVal.getValueType();
8869 WideMaskVT =
8870 EVT::getVectorVT(*DAG.getContext(), MaskVT.getVectorElementType(),
8871 WideVT.getVectorElementCount());
8872 } else {
8873 WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
8874
8875 EVT ValueVT = StVal.getValueType();
8876 WideVT = EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),
8877 WideMaskVT.getVectorElementCount());
8878 }
8879
8880 if (TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) &&
8881 TLI.isTypeLegal(WideMaskVT) && !MST->isCompressingStore()) {
8882 Mask = DAG.getInsertSubvector(dl, DAG.getPOISON(WideMaskVT), Mask, 0);
8883 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
8885 return DAG.getStoreVP(MST->getChain(), dl, StVal, MST->getBasePtr(),
8886 MST->getOffset(), Mask, EVL, MST->getMemoryVT(),
8887 MST->getMemOperand(), MST->getAddressingMode());
8888 }
8889
8890 if (OpNo == 1) {
8891 // The mask should be widened as well.
8892 Mask = ModifyToType(Mask, WideMaskVT, true);
8893 } else {
8894 // Widen the mask.
8895 Mask = ModifyToType(Mask, WideMaskVT, true);
8896
8897 StVal = ModifyToType(StVal, WideVT);
8898 }
8899
8900 assert(Mask.getValueType().getVectorElementCount() ==
8902 "Mask and data vectors should have the same number of elements");
8903 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
8904 MST->getOffset(), Mask, MST->getMemoryVT(),
8905 MST->getMemOperand(), MST->getAddressingMode(),
8906 false, MST->isCompressingStore());
8907}
8908
8909SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8910 assert(OpNo == 4 && "Can widen only the index of mgather");
8911 auto *MG = cast<MaskedGatherSDNode>(N);
8912 SDValue DataOp = MG->getPassThru();
8913 SDValue Mask = MG->getMask();
8914 SDValue Scale = MG->getScale();
8915
8916 // Just widen the index. It's allowed to have extra elements.
8917 SDValue Index = GetWidenedVector(MG->getIndex());
8918
8919 SDLoc dl(N);
8920 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8921 Scale};
8922 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
8923 MG->getMemOperand(), MG->getIndexType(),
8924 MG->getExtensionType());
8925 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
8926 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
8927 return SDValue();
8928}
8929
8930SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8931 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
8932 SDValue DataOp = MSC->getValue();
8933 SDValue Mask = MSC->getMask();
8934 SDValue Index = MSC->getIndex();
8935 SDValue Scale = MSC->getScale();
8936 EVT WideMemVT = MSC->getMemoryVT();
8937
8938 if (OpNo == 1) {
8939 DataOp = GetWidenedVector(DataOp);
8940 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8941
8942 // Widen index.
8943 EVT IndexVT = Index.getValueType();
8944 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
8945 IndexVT.getVectorElementType(), WideEC);
8946 Index = ModifyToType(Index, WideIndexVT);
8947
8948 // The mask should be widened as well.
8949 EVT MaskVT = Mask.getValueType();
8950 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
8951 MaskVT.getVectorElementType(), WideEC);
8952 Mask = ModifyToType(Mask, WideMaskVT, true);
8953
8954 // Widen the MemoryType
8955 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
8956 MSC->getMemoryVT().getScalarType(), WideEC);
8957 } else if (OpNo == 4) {
8958 // Just widen the index. It's allowed to have extra elements.
8959 Index = GetWidenedVector(Index);
8960 } else
8961 llvm_unreachable("Can't widen this operand of mscatter");
8962
8963 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8964 Scale};
8965 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N),
8966 Ops, MSC->getMemOperand(), MSC->getIndexType(),
8967 MSC->isTruncatingStore());
8968}
8969
8970SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8971 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(N);
8972 SDValue DataOp = VPSC->getValue();
8973 SDValue Mask = VPSC->getMask();
8974 SDValue Index = VPSC->getIndex();
8975 SDValue Scale = VPSC->getScale();
8976 EVT WideMemVT = VPSC->getMemoryVT();
8977
8978 if (OpNo == 1) {
8979 DataOp = GetWidenedVector(DataOp);
8980 Index = GetWidenedVector(Index);
8981 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8982 Mask = GetWidenedMask(Mask, WideEC);
8983 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
8984 VPSC->getMemoryVT().getScalarType(), WideEC);
8985 } else if (OpNo == 3) {
8986 // Just widen the index. It's allowed to have extra elements.
8987 Index = GetWidenedVector(Index);
8988 } else
8989 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8990
8991 SDValue Ops[] = {
8992 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8993 VPSC->getVectorLength()};
8994 return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops,
8995 VPSC->getMemOperand(), VPSC->getIndexType());
8996}
8997
8998SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8999 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
9000 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
9001 SDLoc dl(N);
9002 EVT VT = N->getValueType(0);
9003
9004 // WARNING: In this code we widen the compare instruction with garbage.
9005 // This garbage may contain denormal floats which may be slow. Is this a real
9006 // concern ? Should we zero the unused lanes if this is a float compare ?
9007
9008 // Get a new SETCC node to compare the newly widened operands.
9009 // Only some of the compared elements are legal.
9010 EVT SVT = getSetCCResultType(InOp0.getValueType());
9011 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
9012 if (VT.getScalarType() == MVT::i1)
9013 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
9014 SVT.getVectorElementCount());
9015
9016 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
9017 SVT, InOp0, InOp1, N->getOperand(2));
9018
9019 // Extract the needed results from the result vector.
9020 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
9023 SDValue CC = DAG.getExtractSubvector(dl, ResVT, WideSETCC, 0);
9024
9025 EVT OpVT = N->getOperand(0).getValueType();
9026 ISD::NodeType ExtendCode =
9027 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
9028 return DAG.getNode(ExtendCode, dl, VT, CC);
9029}
9030
9031SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
9032 SDValue Chain = N->getOperand(0);
9033 SDValue LHS = GetWidenedVector(N->getOperand(1));
9034 SDValue RHS = GetWidenedVector(N->getOperand(2));
9035 SDValue CC = N->getOperand(3);
9036 SDLoc dl(N);
9037
9038 EVT VT = N->getValueType(0);
9039 EVT EltVT = VT.getVectorElementType();
9040 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
9041 unsigned NumElts = VT.getVectorNumElements();
9042
9043 // Unroll into a build vector.
9044 SmallVector<SDValue, 8> Scalars(NumElts);
9045 SmallVector<SDValue, 8> Chains(NumElts);
9046
9047 for (unsigned i = 0; i != NumElts; ++i) {
9048 SDValue LHSElem = DAG.getExtractVectorElt(dl, TmpEltVT, LHS, i);
9049 SDValue RHSElem = DAG.getExtractVectorElt(dl, TmpEltVT, RHS, i);
9050
9051 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
9052 {Chain, LHSElem, RHSElem, CC});
9053 Chains[i] = Scalars[i].getValue(1);
9054 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
9055 DAG.getBoolConstant(true, dl, EltVT, VT),
9056 DAG.getBoolConstant(false, dl, EltVT, VT));
9057 }
9058
9059 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
9060 ReplaceValueWith(SDValue(N, 1), NewChain);
9061
9062 return DAG.getBuildVector(VT, dl, Scalars);
9063}
9064
9065static unsigned getExtendForIntVecReduction(unsigned Opc) {
9066 switch (Opc) {
9067 default:
9068 llvm_unreachable("Expected integer vector reduction");
9069 case ISD::VECREDUCE_ADD:
9070 case ISD::VECREDUCE_MUL:
9071 case ISD::VECREDUCE_AND:
9072 case ISD::VECREDUCE_OR:
9073 case ISD::VECREDUCE_XOR:
9074 return ISD::ANY_EXTEND;
9077 return ISD::SIGN_EXTEND;
9080 return ISD::ZERO_EXTEND;
9081 }
9082}
9083
9084SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
9085 SDLoc dl(N);
9086 SDValue Op = GetWidenedVector(N->getOperand(0));
9087 EVT VT = N->getValueType(0);
9088 EVT OrigVT = N->getOperand(0).getValueType();
9089 EVT WideVT = Op.getValueType();
9090 EVT ElemVT = OrigVT.getVectorElementType();
9091 SDNodeFlags Flags = N->getFlags();
9092
9093 unsigned Opc = N->getOpcode();
9094 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
9095 SDValue NeutralElem = DAG.getIdentityElement(BaseOpc, dl, ElemVT, Flags);
9096 assert(NeutralElem && "Neutral element must exist");
9097
9098 // Pad the vector with the neutral element.
9099 unsigned OrigElts = OrigVT.getVectorMinNumElements();
9100 unsigned WideElts = WideVT.getVectorMinNumElements();
9101
9102 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
9103 // needing to pad the source vector, because the inactive lanes can simply be
9104 // disabled and not contribute to the result.
9105 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opc);
9106 VPOpcode && TLI.isOperationLegalOrCustom(*VPOpcode, WideVT)) {
9107 SDValue Start = NeutralElem;
9108 if (VT.isInteger())
9109 Start = DAG.getNode(getExtendForIntVecReduction(Opc), dl, VT, Start);
9110 assert(Start.getValueType() == VT);
9111 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
9112 WideVT.getVectorElementCount());
9113 SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
9114 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
9115 OrigVT.getVectorElementCount());
9116 return DAG.getNode(*VPOpcode, dl, VT, {Start, Op, Mask, EVL}, Flags);
9117 }
9118
9119 if (WideVT.isScalableVector()) {
9120 unsigned GCD = std::gcd(OrigElts, WideElts);
9121 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
9123 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
9124 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
9125 Op = DAG.getInsertSubvector(dl, Op, SplatNeutral, Idx);
9126 return DAG.getNode(Opc, dl, VT, Op, Flags);
9127 }
9128
9129 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
9130 Op = DAG.getInsertVectorElt(dl, Op, NeutralElem, Idx);
9131
9132 return DAG.getNode(Opc, dl, VT, Op, Flags);
9133}
9134
9135SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
9136 SDLoc dl(N);
9137 SDValue AccOp = N->getOperand(0);
9138 SDValue VecOp = N->getOperand(1);
9139 SDValue Op = GetWidenedVector(VecOp);
9140
9141 EVT VT = N->getValueType(0);
9142 EVT OrigVT = VecOp.getValueType();
9143 EVT WideVT = Op.getValueType();
9144 EVT ElemVT = OrigVT.getVectorElementType();
9145 SDNodeFlags Flags = N->getFlags();
9146
9147 unsigned Opc = N->getOpcode();
9148 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
9149 SDValue NeutralElem = DAG.getIdentityElement(BaseOpc, dl, ElemVT, Flags);
9150
9151 // Pad the vector with the neutral element.
9152 unsigned OrigElts = OrigVT.getVectorMinNumElements();
9153 unsigned WideElts = WideVT.getVectorMinNumElements();
9154
9155 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
9156 // needing to pad the source vector, because the inactive lanes can simply be
9157 // disabled and not contribute to the result.
9158 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opc);
9159 VPOpcode && TLI.isOperationLegalOrCustom(*VPOpcode, WideVT)) {
9160 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
9161 WideVT.getVectorElementCount());
9162 SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
9163 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
9164 OrigVT.getVectorElementCount());
9165 return DAG.getNode(*VPOpcode, dl, VT, {AccOp, Op, Mask, EVL}, Flags);
9166 }
9167
9168 if (WideVT.isScalableVector()) {
9169 unsigned GCD = std::gcd(OrigElts, WideElts);
9170 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
9172 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
9173 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
9174 Op = DAG.getInsertSubvector(dl, Op, SplatNeutral, Idx);
9175 return DAG.getNode(Opc, dl, VT, AccOp, Op, Flags);
9176 }
9177
9178 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
9179 Op = DAG.getInsertVectorElt(dl, Op, NeutralElem, Idx);
9180
9181 return DAG.getNode(Opc, dl, VT, AccOp, Op, Flags);
9182}
9183
9184SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
9185 assert(N->isVPOpcode() && "Expected VP opcode");
9186
9187 SDLoc dl(N);
9188 SDValue Op = GetWidenedVector(N->getOperand(1));
9189 SDValue Mask = GetWidenedMask(N->getOperand(2),
9190 Op.getValueType().getVectorElementCount());
9191
9192 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0),
9193 {N->getOperand(0), Op, Mask, N->getOperand(3)},
9194 N->getFlags());
9195}
9196
9197SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
9198 // This only gets called in the case that the left and right inputs and
9199 // result are of a legal odd vector type, and the condition is illegal i1 of
9200 // the same odd width that needs widening.
9201 EVT VT = N->getValueType(0);
9202 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
9203
9204 SDValue Cond = GetWidenedVector(N->getOperand(0));
9205 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
9206 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
9207 SDLoc DL(N);
9208
9209 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
9210 LeftIn, RightIn);
9211 return DAG.getExtractSubvector(DL, VT, Select, 0);
9212}
9213
9214SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
9215 SDLoc DL(N);
9216 SDValue Source = N->getOperand(0);
9217 EVT SourceVT = Source.getValueType();
9218 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT);
9219
9220 SDValue WideSource;
9221 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
9222 WideSource = GetWidenedVector(Source);
9223 } else {
9224 // Pad the widened portion with all-ones so the extra lanes appear as
9225 // active (non-zero) elements and do not contribute trailing zeros.
9226 SDValue AllOnes = DAG.getAllOnesConstant(DL, WideVT);
9227 if (WideVT.isFixedLengthVector() &&
9228 getTypeAction(WideVT) == TargetLowering::TypeSplitVector) {
9229 WideSource = GetWidenedVector(Source);
9230 unsigned WideElts = WideVT.getVectorNumElements();
9231 SmallVector<int> Mask(WideElts);
9232 std::iota(Mask.begin(), Mask.end(), 0);
9233 for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
9234 Mask[I] += WideElts;
9235 WideSource = DAG.getVectorShuffle(WideVT, DL, WideSource, AllOnes, Mask);
9236 } else {
9237 WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0);
9238 }
9239 }
9240
9241 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), WideSource,
9242 N->getFlags());
9243}
9244
9245SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
9246 SDLoc DL(N);
9247 SDValue Source = GetWidenedVector(N->getOperand(0));
9248 EVT SrcVT = Source.getValueType();
9249 SDValue Mask =
9250 GetWidenedMask(N->getOperand(1), SrcVT.getVectorElementCount());
9251
9252 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0),
9253 {Source, Mask, N->getOperand(2)}, N->getFlags());
9254}
9255
9256SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
9257 SDLoc DL(N);
9258 SDValue Mask = N->getOperand(0);
9259 EVT OrigMaskVT = Mask.getValueType();
9260 SDValue WideMask = GetWidenedVector(Mask);
9261 EVT WideMaskVT = WideMask.getValueType();
9262
9263 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
9264 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
9265 unsigned WideElts = WideMaskVT.getVectorNumElements();
9266 if (OrigElts != WideElts) {
9267 SDValue ZeroMask = DAG.getConstant(0, DL, WideMaskVT);
9268 WideMask = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideMaskVT, ZeroMask,
9269 Mask, DAG.getVectorIdxConstant(0, DL));
9270 }
9271
9272 return DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, DL, N->getValueType(0),
9273 WideMask);
9274}
9275
9276SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_SHUFFLE_VAR(SDNode *N,
9277 unsigned OpNo) {
9278 // The only possibility for an illegal operand is the mask, since result type
9279 // legalization would have handled this node already otherwise.
9280 assert(OpNo == 1 && "Illegal operand must be mask");
9281 // TODO: Legalize the mask type instead once a target lowers the node.
9282 return TLI.expandVECTOR_SHUFFLE_VAR(N, DAG);
9283}
9284
9285SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
9286 if (OpNo == 0) {
9287 SDLoc DL(N);
9288 EVT ResVT = N->getValueType(0);
9289 EVT SourceVT = N->getOperand(0).getValueType();
9290 EVT WideSourceVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT);
9291 EVT WidenVT =
9292 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
9293 WideSourceVT.getVectorElementCount());
9294
9295 SDValue WideSource = DAG.getInsertSubvector(DL, DAG.getUNDEF(WideSourceVT),
9296 N->getOperand(0), 0);
9297 SDValue WideMask = DAG.getInsertSubvector(
9298 DL, DAG.getConstant(0, DL, WidenVT), N->getOperand(2), 0);
9299 SDValue WideMatch = DAG.getNode(ISD::VECTOR_MATCH, DL, WidenVT, WideSource,
9300 N->getOperand(1), WideMask, N->getFlags());
9301 return DAG.getExtractSubvector(DL, ResVT, WideMatch, 0);
9302 }
9303
9304 // Note: The Mask (OpNo == 2) should be widened with the result.
9305 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
9306
9307 SDLoc DL(N);
9308 SDValue Needle = N->getOperand(1);
9309 EVT NeedleVT = Needle.getValueType();
9310 if (NeedleVT.getVectorNumElements() == 1)
9311 return TLI.expandVectorMatch(N, DAG);
9312
9313 EVT WidenNeedleVT = TLI.getTypeToTransformTo(*DAG.getContext(), NeedleVT);
9314
9315 SDValue Fill =
9316 DAG.getExtractVectorElt(DL, NeedleVT.getVectorElementType(), Needle, 0);
9317 SDValue WideNeedle = DAG.getSplatVector(WidenNeedleVT, DL, Fill);
9318 WideNeedle = DAG.getInsertSubvector(DL, WideNeedle, Needle, 0);
9319
9320 return DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0),
9321 N->getOperand(0), WideNeedle, N->getOperand(2),
9322 N->getFlags());
9323}
9324
9325SDValue DAGTypeLegalizer::WidenVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
9326 // The accumulator is legal and only the inputs need widening. Expand, as
9327 // when widening the result; the extended inputs are widened from there.
9328 return TLI.expandPartialReduceMLA(N, DAG);
9329}
9330
9331//===----------------------------------------------------------------------===//
9332// Vector Widening Utilities
9333//===----------------------------------------------------------------------===//
9334
9335// Utility function to find the type to chop up a widen vector for load/store
9336// TLI: Target lowering used to determine legal types.
9337// Width: Width left need to load/store.
9338// WidenVT: The widen vector type to load to/store from
9339// Align: If 0, don't allow use of a wider type
9340// WidenEx: If Align is not 0, the amount additional we can load/store from.
9341
9342static std::optional<EVT> findMemType(SelectionDAG &DAG,
9343 const TargetLowering &TLI, unsigned Width,
9344 EVT WidenVT, unsigned Align = 0,
9345 unsigned WidenEx = 0) {
9346 EVT WidenEltVT = WidenVT.getVectorElementType();
9347 const bool Scalable = WidenVT.isScalableVector();
9348 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
9349 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
9350 unsigned AlignInBits = Align*8;
9351
9352 EVT RetVT = WidenEltVT;
9353 // Don't bother looking for an integer type if the vector is scalable, skip
9354 // to vector types.
9355 if (!Scalable) {
9356 // If we have one element to load/store, return it.
9357 if (Width == WidenEltWidth)
9358 return RetVT;
9359
9360 // See if there is larger legal integer than the element type to load/store.
9361 for (EVT MemVT : reverse(MVT::integer_valuetypes())) {
9362 unsigned MemVTWidth = MemVT.getSizeInBits();
9363 if (MemVT.getSizeInBits() <= WidenEltWidth)
9364 break;
9365 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
9366 if ((Action == TargetLowering::TypeLegal ||
9368 (WidenWidth % MemVTWidth) == 0 &&
9369 isPowerOf2_32(WidenWidth / MemVTWidth) &&
9370 (MemVTWidth <= Width ||
9371 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9372 if (MemVTWidth == WidenWidth)
9373 return MemVT;
9374 RetVT = MemVT;
9375 break;
9376 }
9377 }
9378 }
9379
9380 // See if there is a larger vector type to load/store that has the same vector
9381 // element type and is evenly divisible with the WidenVT.
9382 for (EVT MemVT : reverse(MVT::vector_valuetypes())) {
9383 // Skip vector MVTs which don't match the scalable property of WidenVT.
9384 if (Scalable != MemVT.isScalableVector())
9385 continue;
9386 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
9387 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
9388 if ((Action == TargetLowering::TypeLegal ||
9390 WidenEltVT == MemVT.getVectorElementType() &&
9391 (WidenWidth % MemVTWidth) == 0 &&
9392 isPowerOf2_32(WidenWidth / MemVTWidth) &&
9393 (MemVTWidth <= Width ||
9394 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9395 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
9396 return MemVT;
9397 }
9398 }
9399
9400 // Using element-wise loads and stores for widening operations is not
9401 // supported for scalable vectors
9402 if (Scalable)
9403 return std::nullopt;
9404
9405 return RetVT;
9406}
9407
9408// Builds a vector type from scalar loads
9409// VecTy: Resulting Vector type
9410// LDOps: Load operators to build a vector type
9411// [Start,End) the list of loads to use.
9414 unsigned Start, unsigned End) {
9415 SDLoc dl(LdOps[Start]);
9416 EVT LdTy = LdOps[Start].getValueType();
9417 unsigned Width = VecTy.getSizeInBits();
9418 unsigned NumElts = Width / LdTy.getSizeInBits();
9419 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
9420
9421 unsigned Idx = 1;
9422 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
9423
9424 for (unsigned i = Start + 1; i != End; ++i) {
9425 EVT NewLdTy = LdOps[i].getValueType();
9426 if (NewLdTy != LdTy) {
9427 NumElts = Width / NewLdTy.getSizeInBits();
9428 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
9429 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
9430 // Readjust position and vector position based on new load type.
9431 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
9432 LdTy = NewLdTy;
9433 }
9434 VecOp = DAG.getInsertVectorElt(dl, VecOp, LdOps[i], Idx++);
9435 }
9436 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
9437}
9438
9439SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
9440 LoadSDNode *LD) {
9441 // The strategy assumes that we can efficiently load power-of-two widths.
9442 // The routine chops the vector into the largest vector loads with the same
9443 // element type or scalar loads and then recombines it to the widen vector
9444 // type.
9445 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
9446 EVT LdVT = LD->getMemoryVT();
9447 SDLoc dl(LD);
9448 assert(LdVT.isVector() && WidenVT.isVector());
9449 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9451
9452 // Load information
9453 SDValue Chain = LD->getChain();
9454 SDValue BasePtr = LD->getBasePtr();
9455 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9456 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9457
9458 TypeSize LdWidth = LdVT.getSizeInBits();
9459 TypeSize WidenWidth = WidenVT.getSizeInBits();
9460 TypeSize WidthDiff = WidenWidth - LdWidth;
9461 // Allow wider loads if they are sufficiently aligned to avoid memory faults
9462 // and if the original load is simple.
9463 unsigned LdAlign =
9464 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
9465
9466 // Find the vector type that can load from.
9467 std::optional<EVT> FirstVT =
9468 findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT, LdAlign,
9469 WidthDiff.getKnownMinValue());
9470
9471 if (!FirstVT)
9472 return SDValue();
9473
9474 SmallVector<EVT, 8> MemVTs;
9475 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
9476
9477 // Unless we're able to load in one instruction we must work out how to load
9478 // the remainder.
9479 if (!TypeSize::isKnownLE(LdWidth, FirstVTWidth)) {
9480 std::optional<EVT> NewVT = FirstVT;
9481 TypeSize RemainingWidth = LdWidth;
9482 TypeSize NewVTWidth = FirstVTWidth;
9483 do {
9484 RemainingWidth -= NewVTWidth;
9485 if (TypeSize::isKnownLT(RemainingWidth, NewVTWidth)) {
9486 // The current type we are using is too large. Find a better size.
9487 NewVT = findMemType(DAG, TLI, RemainingWidth.getKnownMinValue(),
9488 WidenVT, LdAlign, WidthDiff.getKnownMinValue());
9489 if (!NewVT)
9490 return SDValue();
9491 NewVTWidth = NewVT->getSizeInBits();
9492 }
9493 MemVTs.push_back(*NewVT);
9494 } while (TypeSize::isKnownGT(RemainingWidth, NewVTWidth));
9495 }
9496
9497 SDValue LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr, LD->getPointerInfo(),
9498 LD->getBaseAlign(), MMOFlags, Metadata);
9499 LdChain.push_back(LdOp.getValue(1));
9500
9501 // Check if we can load the element with one instruction.
9502 if (MemVTs.empty())
9503 return coerceLoadedValue(LdOp, *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
9504 DAG);
9505
9506 // Load vector by using multiple loads from largest vector to scalar.
9508 LdOps.push_back(LdOp);
9509
9510 uint64_t ScaledOffset = 0;
9511 MachinePointerInfo MPI = LD->getPointerInfo();
9512
9513 // First incremement past the first load.
9514 IncrementPointer(cast<LoadSDNode>(LdOp), *FirstVT, MPI, BasePtr,
9515 &ScaledOffset);
9516
9517 for (EVT MemVT : MemVTs) {
9518 Align NewAlign = ScaledOffset == 0
9519 ? LD->getBaseAlign()
9520 : commonAlignment(LD->getAlign(), ScaledOffset);
9521 SDValue L = DAG.getLoad(MemVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags,
9522 Metadata);
9523
9524 LdOps.push_back(L);
9525 LdChain.push_back(L.getValue(1));
9526 IncrementPointer(cast<LoadSDNode>(L), MemVT, MPI, BasePtr, &ScaledOffset);
9527 }
9528
9529 // Build the vector from the load operations.
9530 unsigned End = LdOps.size();
9531 if (!LdOps[0].getValueType().isVector())
9532 // All the loads are scalar loads.
9533 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
9534
9535 // If the load contains vectors, build the vector using concat vector.
9536 // All of the vectors used to load are power-of-2, and the scalar loads can be
9537 // combined to make a power-of-2 vector.
9538 SmallVector<SDValue, 16> ConcatOps(End);
9539 int i = End - 1;
9540 int Idx = End;
9541 EVT LdTy = LdOps[i].getValueType();
9542 // First, combine the scalar loads to a vector.
9543 if (!LdTy.isVector()) {
9544 for (--i; i >= 0; --i) {
9545 LdTy = LdOps[i].getValueType();
9546 if (LdTy.isVector())
9547 break;
9548 }
9549 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
9550 }
9551
9552 ConcatOps[--Idx] = LdOps[i];
9553 for (--i; i >= 0; --i) {
9554 EVT NewLdTy = LdOps[i].getValueType();
9555 if (NewLdTy != LdTy) {
9556 // Create a larger vector.
9557 TypeSize LdTySize = LdTy.getSizeInBits();
9558 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
9559 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
9560 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
9561 unsigned NumOps =
9562 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
9564 unsigned j = 0;
9565 for (; j != End-Idx; ++j)
9566 WidenOps[j] = ConcatOps[Idx+j];
9567 for (; j != NumOps; ++j)
9568 WidenOps[j] = DAG.getPOISON(LdTy);
9569
9570 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
9571 WidenOps);
9572 Idx = End - 1;
9573 LdTy = NewLdTy;
9574 }
9575 ConcatOps[--Idx] = LdOps[i];
9576 }
9577
9578 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9579 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
9580 ArrayRef(&ConcatOps[Idx], End - Idx));
9581
9582 // We need to fill the rest with undefs to build the vector.
9583 unsigned NumOps =
9584 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9586 SDValue UndefVal = DAG.getPOISON(LdTy);
9587 {
9588 unsigned i = 0;
9589 for (; i != End-Idx; ++i)
9590 WidenOps[i] = ConcatOps[Idx+i];
9591 for (; i != NumOps; ++i)
9592 WidenOps[i] = UndefVal;
9593 }
9594 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
9595}
9596
9597SDValue
9598DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9599 LoadSDNode *LD,
9600 ISD::LoadExtType ExtType) {
9601 // For extension loads, it may not be more efficient to chop up the vector
9602 // and then extend it. Instead, we unroll the load and build a new vector.
9603 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
9604 EVT LdVT = LD->getMemoryVT();
9605 SDLoc dl(LD);
9606 assert(LdVT.isVector() && WidenVT.isVector());
9607 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9608
9609 // Load information
9610 SDValue Chain = LD->getChain();
9611 SDValue BasePtr = LD->getBasePtr();
9612 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9613 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9614
9615 if (LdVT.isScalableVector())
9616 return SDValue();
9617
9618 EVT EltVT = WidenVT.getVectorElementType();
9619 EVT LdEltVT = LdVT.getVectorElementType();
9620 unsigned NumElts = LdVT.getVectorNumElements();
9621
9622 // Load each element and widen.
9623 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9624 SmallVector<SDValue, 16> Ops(WidenNumElts);
9625 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9626 Ops[0] =
9627 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
9628 LdEltVT, LD->getBaseAlign(), MMOFlags, Metadata);
9629 LdChain.push_back(Ops[0].getValue(1));
9630 unsigned i = 0, Offset = Increment;
9631 for (i=1; i < NumElts; ++i, Offset += Increment) {
9632 SDValue NewBasePtr =
9633 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::getFixed(Offset));
9634 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
9635 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
9636 LD->getBaseAlign(), MMOFlags, Metadata);
9637 LdChain.push_back(Ops[i].getValue(1));
9638 }
9639
9640 // Fill the rest with undefs.
9641 SDValue UndefVal = DAG.getPOISON(EltVT);
9642 for (; i != WidenNumElts; ++i)
9643 Ops[i] = UndefVal;
9644
9645 return DAG.getBuildVector(WidenVT, dl, Ops);
9646}
9647
9648bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9649 StoreSDNode *ST) {
9650 // The strategy assumes that we can efficiently store power-of-two widths.
9651 // The routine chops the vector into the largest vector stores with the same
9652 // element type or scalar stores.
9653 SDValue Chain = ST->getChain();
9654 SDValue BasePtr = ST->getBasePtr();
9655 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9656 MMOMetadata Metadata = ST->getMMOMetadataForSubAccess();
9657 SDValue ValOp = GetWidenedVector(ST->getValue());
9658 SDLoc dl(ST);
9659
9660 EVT StVT = ST->getMemoryVT();
9661 TypeSize StWidth = StVT.getSizeInBits();
9662 EVT ValVT = ValOp.getValueType();
9663 TypeSize ValWidth = ValVT.getSizeInBits();
9664 EVT ValEltVT = ValVT.getVectorElementType();
9665 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9666 assert(StVT.getVectorElementType() == ValEltVT);
9667 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9668 "Mismatch between store and value types");
9669
9670 int Idx = 0; // current index to store
9671
9672 MachinePointerInfo MPI = ST->getPointerInfo();
9673 uint64_t ScaledOffset = 0;
9674
9675 // A breakdown of how to widen this vector store. Each element of the vector
9676 // is a memory VT combined with the number of times it is to be stored to,
9677 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9679
9680 while (StWidth.isNonZero()) {
9681 // Find the largest vector type we can store with.
9682 std::optional<EVT> NewVT =
9683 findMemType(DAG, TLI, StWidth.getKnownMinValue(), ValVT);
9684 if (!NewVT)
9685 return false;
9686 MemVTs.push_back({*NewVT, 0});
9687 TypeSize NewVTWidth = NewVT->getSizeInBits();
9688
9689 do {
9690 StWidth -= NewVTWidth;
9691 MemVTs.back().second++;
9692 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
9693 }
9694
9695 for (const auto &Pair : MemVTs) {
9696 EVT NewVT = Pair.first;
9697 unsigned Count = Pair.second;
9698 TypeSize NewVTWidth = NewVT.getSizeInBits();
9699
9700 if (NewVT.isVector()) {
9701 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9702 do {
9703 Align NewAlign = ScaledOffset == 0
9704 ? ST->getBaseAlign()
9705 : commonAlignment(ST->getAlign(), ScaledOffset);
9706 SDValue EOp = DAG.getExtractSubvector(dl, NewVT, ValOp, Idx);
9707 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign,
9708 MMOFlags, Metadata);
9709 StChain.push_back(PartStore);
9710
9711 Idx += NumVTElts;
9712 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr,
9713 &ScaledOffset);
9714 } while (--Count);
9715 } else {
9716 // Cast the vector to the scalar type we can store.
9717 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9718 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
9719 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
9720 // Readjust index position based on new vector type.
9721 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9722 do {
9723 SDValue EOp = DAG.getExtractVectorElt(dl, NewVT, VecOp, Idx++);
9724 SDValue PartStore =
9725 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getBaseAlign(),
9726 MMOFlags, Metadata);
9727 StChain.push_back(PartStore);
9728
9729 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr);
9730 } while (--Count);
9731 // Restore index back to be relative to the original widen element type.
9732 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9733 }
9734 }
9735
9736 return true;
9737}
9738
9739/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9740/// input vector must have the same element type as NVT.
9741/// FillWithZeroes specifies that the vector should be widened with zeroes.
9742SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9743 bool FillWithZeroes) {
9744 // Note that InOp might have been widened so it might already have
9745 // the right width or it might need be narrowed.
9746 EVT InVT = InOp.getValueType();
9748 "input and widen element type must match");
9749 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9750 "cannot modify scalable vectors in this way");
9751 SDLoc dl(InOp);
9752
9753 // Check if InOp already has the right width.
9754 if (InVT == NVT)
9755 return InOp;
9756
9757 ElementCount InEC = InVT.getVectorElementCount();
9758 ElementCount WidenEC = NVT.getVectorElementCount();
9759 if (WidenEC.hasKnownScalarFactor(InEC)) {
9760 unsigned NumConcat = WidenEC.getKnownScalarFactor(InEC);
9761 SmallVector<SDValue, 16> Ops(NumConcat);
9762 SDValue FillVal =
9763 FillWithZeroes ? DAG.getConstant(0, dl, InVT) : DAG.getPOISON(InVT);
9764 Ops[0] = InOp;
9765 for (unsigned i = 1; i != NumConcat; ++i)
9766 Ops[i] = FillVal;
9767
9768 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
9769 }
9770
9771 if (InEC.hasKnownScalarFactor(WidenEC))
9772 return DAG.getExtractSubvector(dl, NVT, InOp, 0);
9773
9774 if (NVT.isScalableVector() && InVT.isScalableVector()) {
9775 // Split the input into the largest equal-sized scalable subvectors.
9776 unsigned InNumElts = InVT.getVectorMinNumElements();
9777 unsigned NewNumElts = NVT.getVectorMinNumElements();
9778 unsigned CommonFactor = std::gcd(InNumElts, NewNumElts);
9779 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
9780 ElementCount::getScalable(CommonFactor));
9781
9783 unsigned NumCopiedParts = std::min(InNumElts, NewNumElts) / CommonFactor;
9784 for (unsigned I = 0; I != NumCopiedParts; ++I)
9785 Ops.push_back(
9786 DAG.getExtractSubvector(dl, PartVT, InOp, I * CommonFactor));
9787
9788 unsigned NumResultParts = NewNumElts / CommonFactor;
9789 if (NumResultParts > NumCopiedParts) {
9790 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, PartVT)
9791 : DAG.getPOISON(PartVT);
9792 Ops.append(NumResultParts - NumCopiedParts, FillVal);
9793 }
9794
9795 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
9796 }
9797
9798 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9799 "Scalable vectors should have been handled already.");
9800
9801 unsigned InNumElts = InEC.getFixedValue();
9802 unsigned WidenNumElts = WidenEC.getFixedValue();
9803
9804 // Fall back to extract and build (+ mask, if padding with zeros).
9805 SmallVector<SDValue, 16> Ops(WidenNumElts);
9806 EVT EltVT = NVT.getVectorElementType();
9807 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
9808 unsigned Idx;
9809 for (Idx = 0; Idx < MinNumElts; ++Idx)
9810 Ops[Idx] = DAG.getExtractVectorElt(dl, EltVT, InOp, Idx);
9811
9812 SDValue UndefVal = DAG.getPOISON(EltVT);
9813 for (; Idx < WidenNumElts; ++Idx)
9814 Ops[Idx] = UndefVal;
9815
9816 SDValue Widened = DAG.getBuildVector(NVT, dl, Ops);
9817 if (!FillWithZeroes)
9818 return Widened;
9819
9820 assert(NVT.isInteger() &&
9821 "We expect to never want to FillWithZeroes for non-integral types.");
9822
9824 MaskOps.append(MinNumElts, DAG.getAllOnesConstant(dl, EltVT));
9825 MaskOps.append(WidenNumElts - MinNumElts, DAG.getConstant(0, dl, EltVT));
9826
9827 return DAG.getNode(ISD::AND, dl, NVT, Widened,
9828 DAG.getBuildVector(NVT, dl, MaskOps));
9829}
return SDValue()
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
dxil translate DXIL Translate Metadata
#define _
static constexpr Value * getValue(Ty &ValueOrUse)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static unsigned getExtendForIntVecReduction(SDNode *N)
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
static std::optional< EVT > findMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align, unsigned WidenEx)
static EVT getSETCCOperandType(SDValue N)
static bool isSETCCOp(unsigned Opcode)
static bool isLogicalMaskOp(unsigned Opcode)
static bool isSETCCorConvertedSETCC(SDValue N)
static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT, TypeSize FirstVTWidth, const SDLoc &dl, SelectionDAG &DAG)
Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the widened value so it can b...
static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, SmallVectorImpl< SDValue > &ConcatOps, unsigned ConcatEnd, EVT VT, EVT MaxVT, EVT WidenVT)
static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT, TypeSize LdWidth, TypeSize FirstVTWidth, SDLoc dl, SelectionDAG &DAG)
Either return the same load or provide appropriate casts from the load and return that.
#define I(x, y, z)
Definition MD5.cpp:57
static bool isUndef(const MachineInstr &MI)
This file provides utility analysis objects describing memory locations.
uint64_t High
#define P(N)
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
const MachineOperand & HiOp1
SI Fold Operands
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
This file implements the SmallBitVector class.
#define LLVM_DEBUG(...)
Definition Debug.h:119
Value * RHS
Value * LHS
This is an SDNode representing atomic operations.
LLVM_ABI unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
bool isValid() const
Return true if the attribute is any kind of attribute.
Definition Attributes.h:263
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition TypeSize.h:309
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:312
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
This class is used to represent ISD::LOAD nodes.
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
static auto integer_valuetypes()
static auto vector_valuetypes()
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Flags
Flags values. These may be or'd together.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
This class is used to represent an MGATHER node.
const SDValue & getBasePtr() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getInc() const
const SDValue & getScale() const
const SDValue & getMask() const
const SDValue & getIntID() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
ISD::MemIndexType getIndexType() const
This class is used to represent an MLOAD node.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
const SDValue & getMask() const
const SDValue & getPassThru() const
const SDValue & getOffset() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getOffset() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
This is an abstract virtual class for memory operations.
Align getBaseAlign() const
Returns alignment and volatility of the memory access.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
const MDNode * getMemCacheHint() const
Returns the cache hint metadata for this memory access.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDNodeFlags getFlags() const
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
SDValue getInsertVectorElt(const SDLoc &DL, SDValue Vec, SDValue Elt, unsigned Idx)
Insert Elt into Vec at offset Idx.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVMContext * getContext() const
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition SetVector.h:94
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
BooleanContent
Enum that describes how the target represents true/false values.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
This class is used to represent an VP_GATHER node.
const SDValue & getScale() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getVectorLength() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
This class is used to represent a VP_LOAD node.
const SDValue & getValue() const
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
const SDValue & getMask() const
ISD::LoadExtType getExtensionType() const
const SDValue & getStride() const
const SDValue & getOffset() const
const SDValue & getVectorLength() const
const SDValue & getBasePtr() const
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if this is a truncating store.
const SDValue & getOffset() const
const SDValue & getVectorLength() const
const SDValue & getStride() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
Definition TypeSize.h:181
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
Definition TypeSize.h:266
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isNonZero() const
Definition TypeSize.h:156
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS*X will result in a value whose quantity matches our own.
Definition TypeSize.h:274
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:217
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
Definition TypeSize.h:177
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr bool isZero() const
Definition TypeSize.h:154
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:224
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:238
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ ATOMIC_LOAD_FMINIMUMNUM
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ VECREDUCE_FMINIMUMNUM
@ AVGSUBFLOORS
AVGSUBFLOORS/AVGSUBFLOORU - Averaging subtract - Subtract two integers using an integer of type i[N+1...
Definition ISDOpcodes.h:739
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:397
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:403
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ ATOMIC_LOAD_USUB_COND
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
Definition ISDOpcodes.h:665
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition ISDOpcodes.h:942
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
Definition ISDOpcodes.h:520
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:410
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ ATOMIC_LOAD_USUB_SAT
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:731
@ STRICT_UINT_TO_FP
Definition ISDOpcodes.h:490
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PARTIAL_REDUCE_FMLA
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition ISDOpcodes.h:707
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ ATOMIC_LOAD_FMAXIMUM
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
Definition ISDOpcodes.h:80
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:389
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ ATOMIC_LOAD_FMINIMUM
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
Definition ISDOpcodes.h:669
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition ISDOpcodes.h:931
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition ISDOpcodes.h:650
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:416
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition ISDOpcodes.h:489
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ ATOMIC_LOAD_FMAXIMUMNUM
@ ATOMIC_LOAD_UDEC_WRAP
@ STRICT_FP_TO_UINT
Definition ISDOpcodes.h:483
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
Definition ISDOpcodes.h:807
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:482
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition ISDOpcodes.h:510
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:759
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:726
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
Definition ISDOpcodes.h:673
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition ISDOpcodes.h:715
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition ISDOpcodes.h:953
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Experimental vector histogram intrinsic Operands: Input Chain, Inc, Mask, Base, Index,...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ VECREDUCE_FMINIMUM
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ VECREDUCE_SEQ_FMUL
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ ATOMIC_LOAD_UINC_WRAP
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ PARTIAL_REDUCE_SUMLA
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
Definition ISDOpcodes.h:645
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ CTTZ_ELTS_ZERO_POISON
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:746
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
Definition ISDOpcodes.h:775
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
LLVM_ABI NodeType getUnmaskedBinOpOpcode(unsigned MaskedOpc)
Given a MaskedOpc of ISD::MASKED_(U|S)(DIV|REM), returns the unmasked ISD::(U|S)(DIV|REM).
LLVM_ABI bool isBuildVectorAllOnes(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LLVM_ABI std::optional< unsigned > getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
LLVM_ABI LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
constexpr double e
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1781
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr int PoisonMaskElem
@ Mul
Product of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
OutputIt copy(R &&Range, OutputIt Out)
Definition STLExtras.h:1901
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
LLVM_ABI void processShuffleMasks(ArrayRef< int > Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, unsigned NumOfUsedRegs, function_ref< void()> NoInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> SingleInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned, bool)> ManyInputsAction)
Splits and processes shuffle mask depending on the number of input and output registers.
@ Increment
Incrementally increasing token ID.
Definition AllocToken.h:26
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition ValueTypes.h:90
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:509
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:281
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition ValueTypes.h:516
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
Definition ValueTypes.h:490
EVT changeVectorElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element coun...
Definition ValueTypes.h:109
bool isFixedLengthVector() const
Definition ValueTypes.h:214
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition ValueTypes.h:55
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
Definition ValueTypes.h:457
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition ValueTypes.h:294
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
bool knownBitsGE(EVT VT) const
Return true if we know at compile time this has more than or the same bits as VT.
Definition ValueTypes.h:306
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:499
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.