34#define DEBUG_TYPE "legalize-types"
40void DAGTypeLegalizer::ScalarizeVectorResult(
SDNode *
N,
unsigned ResNo) {
41 LLVM_DEBUG(
dbgs() <<
"Scalarize node result " << ResNo <<
": ";
N->dump(&DAG);
45 switch (
N->getOpcode()) {
48 dbgs() <<
"ScalarizeVectorResult #" << ResNo <<
": ";
60 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(
N);
break;
62 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(
N));
break;
68 case ISD::SETCC: R = ScalarizeVecRes_SETCC(
N);
break;
69 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(
N);
break;
75 R = ScalarizeVecRes_VecInregOp(
N);
114 R = ScalarizeVecRes_UnaryOp(
N);
157 R = ScalarizeVecRes_BinOp(
N);
162 R = ScalarizeVecRes_TernaryOp(
N);
165#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
166 case ISD::STRICT_##DAGN:
167#include "llvm/IR/ConstrainedOps.def"
168 R = ScalarizeVecRes_StrictFPOp(
N);
173 R = ScalarizeVecRes_FP_TO_XINT_SAT(
N);
182 R = ScalarizeVecRes_OverflowOp(
N, ResNo);
192 R = ScalarizeVecRes_FIX(
N);
198 SetScalarizedVector(
SDValue(
N, ResNo), R);
202 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
203 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
205 LHS.getValueType(), LHS, RHS,
N->getFlags());
209 SDValue Op0 = GetScalarizedVector(
N->getOperand(0));
210 SDValue Op1 = GetScalarizedVector(
N->getOperand(1));
211 SDValue Op2 = GetScalarizedVector(
N->getOperand(2));
217 SDValue Op0 = GetScalarizedVector(
N->getOperand(0));
218 SDValue Op1 = GetScalarizedVector(
N->getOperand(1));
225 EVT VT =
N->getValueType(0).getVectorElementType();
226 unsigned NumOpers =
N->getNumOperands();
228 EVT ValueVTs[] = {VT, MVT::Other};
237 for (
unsigned i = 1; i < NumOpers; ++i) {
243 Oper = GetScalarizedVector(Oper);
254 Opers,
N->getFlags());
265 EVT ResVT =
N->getValueType(0);
266 EVT OvVT =
N->getValueType(1);
270 ScalarLHS = GetScalarizedVector(
N->getOperand(0));
271 ScalarRHS = GetScalarizedVector(
N->getOperand(1));
276 ScalarLHS = ElemsLHS[0];
277 ScalarRHS = ElemsRHS[0];
283 N->getOpcode(),
DL, ScalarVTs, ScalarLHS, ScalarRHS).
getNode();
287 unsigned OtherNo = 1 - ResNo;
288 EVT OtherVT =
N->getValueType(OtherNo);
290 SetScalarizedVector(
SDValue(
N, OtherNo),
SDValue(ScalarNode, OtherNo));
294 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
297 return SDValue(ScalarNode, ResNo);
302 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
303 return GetScalarizedVector(Op);
308 if (
Op.getValueType().isVector()
309 &&
Op.getValueType().getVectorNumElements() == 1
310 && !isSimpleLegalType(
Op.getValueType()))
311 Op = GetScalarizedVector(Op);
312 EVT NewVT =
N->getValueType(0).getVectorElementType();
317SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(
SDNode *
N) {
318 EVT EltVT =
N->getValueType(0).getVectorElementType();
327SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
329 N->getValueType(0).getVectorElementType(),
330 N->getOperand(0),
N->getOperand(1));
336 EVT OpVT =
Op.getValueType();
340 Op = GetScalarizedVector(Op);
347 N->getValueType(0).getVectorElementType(), Op,
352 SDValue Op = GetScalarizedVector(
N->getOperand(0));
354 Op.getValueType(), Op,
N->getOperand(1));
357SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(
SDNode *
N) {
361 EVT EltVT =
N->getValueType(0).getVectorElementType();
362 if (
Op.getValueType() != EltVT)
369 assert(
N->isUnindexed() &&
"Indexed vector load?");
373 N->getValueType(0).getVectorElementType(),
SDLoc(
N),
N->getChain(),
374 N->getBasePtr(), DAG.
getUNDEF(
N->getBasePtr().getValueType()),
375 N->getPointerInfo(),
N->getMemoryVT().getVectorElementType(),
376 N->getOriginalAlign(),
N->getMemOperand()->getFlags(),
N->getAAInfo());
386 EVT DestVT =
N->getValueType(0).getVectorElementType();
388 EVT OpVT =
Op.getValueType();
398 Op = GetScalarizedVector(Op);
404 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), DestVT, Op,
N->getFlags());
408 EVT EltVT =
N->getValueType(0).getVectorElementType();
410 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
419 EVT OpVT =
Op.getValueType();
421 EVT EltVT =
N->getValueType(0).getVectorElementType();
424 Op = GetScalarizedVector(Op);
430 switch (
N->getOpcode()) {
442SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(
SDNode *
N) {
445 EVT EltVT =
N->getValueType(0).getVectorElementType();
454 EVT OpVT =
Cond.getValueType();
467 SDValue LHS = GetScalarizedVector(
N->getOperand(1));
481 EVT OpVT =
Cond->getOperand(0).getValueType();
488 EVT CondVT =
Cond.getValueType();
489 if (ScalarBool != VecBool) {
490 switch (ScalarBool) {
511 auto BoolVT = getSetCCResultType(CondVT);
512 if (BoolVT.bitsLT(CondVT))
517 GetScalarizedVector(
N->getOperand(2)));
521 SDValue LHS = GetScalarizedVector(
N->getOperand(1));
523 LHS.getValueType(),
N->getOperand(0), LHS,
524 GetScalarizedVector(
N->getOperand(2)));
528 SDValue LHS = GetScalarizedVector(
N->getOperand(2));
530 N->getOperand(0),
N->getOperand(1),
531 LHS, GetScalarizedVector(
N->getOperand(3)),
536 return DAG.
getUNDEF(
N->getValueType(0).getVectorElementType());
539SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(
SDNode *
N) {
543 return DAG.
getUNDEF(
N->getValueType(0).getVectorElementType());
544 unsigned Op = !cast<ConstantSDNode>(
Arg)->isZero();
545 return GetScalarizedVector(
N->getOperand(Op));
548SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(
SDNode *
N) {
550 EVT SrcVT = Src.getValueType();
555 Src = GetScalarizedVector(Src);
561 EVT DstVT =
N->getValueType(0).getVectorElementType();
562 return DAG.
getNode(
N->getOpcode(), dl, DstVT, Src,
N->getOperand(1));
566 assert(
N->getValueType(0).isVector() &&
567 N->getOperand(0).getValueType().isVector() &&
568 "Operand types must be vectors");
571 EVT OpVT =
LHS.getValueType();
572 EVT NVT =
N->getValueType(0).getVectorElementType();
577 LHS = GetScalarizedVector(LHS);
578 RHS = GetScalarizedVector(RHS);
594 return DAG.
getNode(ExtendCode,
DL, NVT, Res);
601 EVT ArgVT =
Arg.getValueType();
602 EVT ResultVT =
N->getValueType(0).getVectorElementType();
605 Arg = GetScalarizedVector(
Arg);
618 return DAG.
getNode(ExtendCode,
DL, ResultVT, Res);
625bool DAGTypeLegalizer::ScalarizeVectorOperand(
SDNode *
N,
unsigned OpNo) {
626 LLVM_DEBUG(
dbgs() <<
"Scalarize node operand " << OpNo <<
": ";
N->dump(&DAG);
630 switch (
N->getOpcode()) {
633 dbgs() <<
"ScalarizeVectorOperand Op #" << OpNo <<
": ";
640 Res = ScalarizeVecOp_BITCAST(
N);
650 Res = ScalarizeVecOp_UnaryOp(
N);
656 Res = ScalarizeVecOp_UnaryOp_StrictFP(
N);
659 Res = ScalarizeVecOp_CONCAT_VECTORS(
N);
662 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(
N);
665 Res = ScalarizeVecOp_VSELECT(
N);
668 Res = ScalarizeVecOp_VSETCC(
N);
671 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(
N), OpNo);
674 Res = ScalarizeVecOp_STRICT_FP_ROUND(
N, OpNo);
677 Res = ScalarizeVecOp_FP_ROUND(
N, OpNo);
680 Res = ScalarizeVecOp_STRICT_FP_EXTEND(
N);
683 Res = ScalarizeVecOp_FP_EXTEND(
N);
698 Res = ScalarizeVecOp_VECREDUCE(
N);
702 Res = ScalarizeVecOp_VECREDUCE_SEQ(
N);
707 if (!Res.getNode())
return false;
711 if (Res.getNode() ==
N)
714 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 1 &&
715 "Invalid operand expansion");
717 ReplaceValueWith(
SDValue(
N, 0), Res);
724 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
726 N->getValueType(0), Elt);
732 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
733 "Unexpected vector type!");
734 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
736 N->getValueType(0).getScalarType(), Elt);
744SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(
SDNode *
N) {
745 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
746 "Unexpected vector type!");
747 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
749 {
N->getValueType(0).getScalarType(), MVT::Other },
750 {
N->getOperand(0), Elt });
760 ReplaceValueWith(
SDValue(
N, 0), Res);
765SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(
SDNode *
N) {
767 for (
unsigned i = 0, e =
N->getNumOperands(); i < e; ++i)
768 Ops[i] = GetScalarizedVector(
N->getOperand(i));
774SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
775 EVT VT =
N->getValueType(0);
776 SDValue Res = GetScalarizedVector(
N->getOperand(0));
788 SDValue ScalarCond = GetScalarizedVector(
N->getOperand(0));
789 EVT VT =
N->getValueType(0);
799 assert(
N->getValueType(0).isVector() &&
800 N->getOperand(0).getValueType().isVector() &&
801 "Operand types must be vectors");
802 assert(
N->getValueType(0) == MVT::v1i1 &&
"Expected v1i1 type");
804 EVT VT =
N->getValueType(0);
805 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
806 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
808 EVT OpVT =
N->getOperand(0).getValueType();
820 Res = DAG.
getNode(ExtendCode,
DL, NVT, Res);
828 assert(
N->isUnindexed() &&
"Indexed store of one-element vector?");
829 assert(OpNo == 1 &&
"Do not know how to scalarize this operand!");
832 if (
N->isTruncatingStore())
834 N->getChain(), dl, GetScalarizedVector(
N->getOperand(1)),
835 N->getBasePtr(),
N->getPointerInfo(),
836 N->getMemoryVT().getVectorElementType(),
N->getOriginalAlign(),
837 N->getMemOperand()->getFlags(),
N->getAAInfo());
839 return DAG.
getStore(
N->getChain(), dl, GetScalarizedVector(
N->getOperand(1)),
840 N->getBasePtr(),
N->getPointerInfo(),
841 N->getOriginalAlign(),
N->getMemOperand()->getFlags(),
847SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(
SDNode *
N,
unsigned OpNo) {
848 assert(OpNo == 0 &&
"Wrong operand for scalarization!");
849 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
851 N->getValueType(0).getVectorElementType(), Elt,
856SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(
SDNode *
N,
858 assert(OpNo == 1 &&
"Wrong operand for scalarization!");
859 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
863 {
N->getOperand(0), Elt,
N->getOperand(2) });
872 ReplaceValueWith(
SDValue(
N, 0), Res);
879 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
881 N->getValueType(0).getVectorElementType(), Elt);
887SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(
SDNode *
N) {
888 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
892 {
N->getOperand(0), Elt});
901 ReplaceValueWith(
SDValue(
N, 0), Res);
906 SDValue Res = GetScalarizedVector(
N->getOperand(0));
913SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(
SDNode *
N) {
921 AccOp, Op,
N->getFlags());
932void DAGTypeLegalizer::SplitVectorResult(
SDNode *
N,
unsigned ResNo) {
937 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
940 switch (
N->getOpcode()) {
943 dbgs() <<
"SplitVectorResult #" << ResNo <<
": ";
955 case ISD::VP_SELECT: SplitRes_Select(
N,
Lo,
Hi);
break;
969 SplitVecRes_ScalarOp(
N,
Lo,
Hi);
972 SplitVecRes_STEP_VECTOR(
N,
Lo,
Hi);
976 SplitVecRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
979 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(
N),
Lo,
Hi);
981 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
982 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(
N),
Lo,
Hi);
985 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(
N),
Lo,
Hi);
989 SplitVecRes_Gather(cast<MemSDNode>(
N),
Lo,
Hi,
true);
993 SplitVecRes_SETCC(
N,
Lo,
Hi);
996 SplitVecRes_VECTOR_REVERSE(
N,
Lo,
Hi);
999 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(
N),
Lo,
Hi);
1002 SplitVecRes_VECTOR_SPLICE(
N,
Lo,
Hi);
1005 SplitVecRes_VECTOR_DEINTERLEAVE(
N);
1008 SplitVecRes_VECTOR_INTERLEAVE(
N);
1011 SplitVecRes_VAARG(
N,
Lo,
Hi);
1017 SplitVecRes_ExtVecInRegOp(
N,
Lo,
Hi);
1023 case ISD::VP_BITREVERSE:
1031 case ISD::VP_CTLZ_ZERO_UNDEF:
1033 case ISD::VP_CTTZ_ZERO_UNDEF:
1043 case ISD::VP_FFLOOR:
1048 case ISD::VP_FNEARBYINT:
1053 case ISD::VP_FP_EXTEND:
1055 case ISD::VP_FP_ROUND:
1057 case ISD::VP_FP_TO_SINT:
1059 case ISD::VP_FP_TO_UINT:
1063 case ISD::VP_FROUND:
1065 case ISD::VP_FROUNDEVEN:
1069 case ISD::VP_FROUNDTOZERO:
1071 case ISD::VP_SINT_TO_FP:
1073 case ISD::VP_TRUNCATE:
1075 case ISD::VP_UINT_TO_FP:
1077 SplitVecRes_UnaryOp(
N,
Lo,
Hi);
1083 case ISD::VP_SIGN_EXTEND:
1084 case ISD::VP_ZERO_EXTEND:
1085 SplitVecRes_ExtendOp(
N,
Lo,
Hi);
1105 case ISD::OR:
case ISD::VP_OR:
1125 case ISD::VP_FCOPYSIGN:
1126 SplitVecRes_BinOp(
N,
Lo,
Hi);
1133 SplitVecRes_TernaryOp(
N,
Lo,
Hi);
1136#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1137 case ISD::STRICT_##DAGN:
1138#include "llvm/IR/ConstrainedOps.def"
1139 SplitVecRes_StrictFPOp(
N,
Lo,
Hi);
1144 SplitVecRes_FP_TO_XINT_SAT(
N,
Lo,
Hi);
1153 SplitVecRes_OverflowOp(
N, ResNo,
Lo,
Hi);
1163 SplitVecRes_FIX(
N,
Lo,
Hi);
1172void DAGTypeLegalizer::IncrementPointer(
MemSDNode *
N,
EVT MemVT,
1181 DL,
Ptr.getValueType(),
1182 APInt(
Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1184 Flags.setNoUnsignedWrap(
true);
1186 *ScaledOffset += IncrementSize;
1190 MPI =
N->getPointerInfo().getWithOffset(IncrementSize);
1196std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(
SDValue Mask) {
1197 return SplitMask(Mask,
SDLoc(Mask));
1200std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(
SDValue Mask,
1203 EVT MaskVT =
Mask.getValueType();
1205 GetSplitVector(Mask, MaskLo, MaskHi);
1208 return std::make_pair(MaskLo, MaskHi);
1213 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1215 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
1219 unsigned Opcode =
N->getOpcode();
1220 if (
N->getNumOperands() == 2) {
1226 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1227 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1230 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(2));
1233 std::tie(EVLLo, EVLHi) =
1234 DAG.
SplitEVL(
N->getOperand(3),
N->getValueType(0), dl);
1237 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1239 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1245 GetSplitVector(
N->getOperand(0), Op0Lo, Op0Hi);
1247 GetSplitVector(
N->getOperand(1), Op1Lo, Op1Hi);
1249 GetSplitVector(
N->getOperand(2), Op2Lo, Op2Hi);
1253 unsigned Opcode =
N->getOpcode();
1254 if (
N->getNumOperands() == 3) {
1260 assert(
N->getNumOperands() == 5 &&
"Unexpected number of operands!");
1261 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1264 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(3));
1267 std::tie(EVLLo, EVLHi) =
1268 DAG.
SplitEVL(
N->getOperand(4),
N->getValueType(0), dl);
1271 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1273 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1278 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1280 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
1284 unsigned Opcode =
N->getOpcode();
1303 switch (getTypeAction(InVT)) {
1318 GetExpandedOp(InOp,
Lo,
Hi);
1329 GetSplitVector(InOp,
Lo,
Hi);
1343 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT,
Lo,
Hi);
1366 assert(!(
N->getNumOperands() & 1) &&
"Unsupported CONCAT_VECTORS");
1368 unsigned NumSubvectors =
N->getNumOperands() / 2;
1369 if (NumSubvectors == 1) {
1370 Lo =
N->getOperand(0);
1371 Hi =
N->getOperand(1);
1385void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(
SDNode *
N,
SDValue &
Lo,
1395 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
1407 GetSplitVector(Vec,
Lo,
Hi);
1410 EVT LoVT =
Lo.getValueType();
1419 unsigned IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
1420 if (IdxVal + SubElems <= LoElems) {
1428 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1454 Lo = DAG.
getLoad(
Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1458 auto *
Load = cast<LoadSDNode>(
Lo);
1460 IncrementPointer(Load, LoVT, MPI, StackPtr);
1463 Hi = DAG.
getLoad(
Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1469 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
1477 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1482 EVT RHSVT =
RHS.getValueType();
1484 GetSplitVector(RHS, RHSLo, RHSHi);
1498 SDValue FpValue =
N->getOperand(0);
1500 GetSplitVector(FpValue, ArgLo, ArgHi);
1513 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1517 std::tie(LoVT, HiVT) =
1528 unsigned Opcode =
N->getOpcode();
1535 GetSplitVector(N0, InLo, InHi);
1542 EVT OutLoVT, OutHiVT;
1545 assert((2 * OutNumElements) <= InNumElements &&
1546 "Illegal extend vector in reg split");
1556 for (
unsigned i = 0; i != OutNumElements; ++i)
1557 SplitHi[i] = i + OutNumElements;
1560 Lo = DAG.
getNode(Opcode, dl, OutLoVT, InLo);
1561 Hi = DAG.
getNode(Opcode, dl, OutHiVT, InHi);
1566 unsigned NumOps =
N->getNumOperands();
1580 for (
unsigned i = 1; i < NumOps; ++i) {
1585 EVT InVT =
Op.getValueType();
1590 GetSplitVector(Op, OpLo, OpHi);
1599 EVT LoValueVTs[] = {LoVT, MVT::Other};
1600 EVT HiValueVTs[] = {HiVT, MVT::Other};
1609 Lo.getValue(1),
Hi.getValue(1));
1613 ReplaceValueWith(
SDValue(
N, 1), Chain);
1616SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(
SDNode *
N,
unsigned ResNE) {
1618 EVT VT =
N->getValueType(0);
1629 else if (NE > ResNE)
1633 EVT ChainVTs[] = {EltVT, MVT::Other};
1637 for (i = 0; i !=
NE; ++i) {
1639 for (
unsigned j = 1, e =
N->getNumOperands(); j != e; ++j) {
1640 SDValue Operand =
N->getOperand(j);
1651 Scalar.getNode()->setFlags(
N->getFlags());
1659 for (; i < ResNE; ++i)
1664 ReplaceValueWith(
SDValue(
N, 1), Chain);
1671void DAGTypeLegalizer::SplitVecRes_OverflowOp(
SDNode *
N,
unsigned ResNo,
1674 EVT ResVT =
N->getValueType(0);
1675 EVT OvVT =
N->getValueType(1);
1676 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1680 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1682 GetSplitVector(
N->getOperand(0), LoLHS, HiLHS);
1683 GetSplitVector(
N->getOperand(1), LoRHS, HiRHS);
1689 unsigned Opcode =
N->getOpcode();
1701 unsigned OtherNo = 1 - ResNo;
1702 EVT OtherVT =
N->getValueType(OtherNo);
1704 SetSplitVector(
SDValue(
N, OtherNo),
1710 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
1714void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(
SDNode *
N,
SDValue &
Lo,
1720 GetSplitVector(Vec,
Lo,
Hi);
1723 unsigned IdxVal = CIdx->getZExtValue();
1724 unsigned LoNumElts =
Lo.getValueType().getVectorMinNumElements();
1725 if (IdxVal < LoNumElts) {
1727 Lo.getValueType(),
Lo, Elt,
Idx);
1737 if (CustomLowerNode(
N,
N->getValueType(0),
true))
1778 Lo = DAG.
getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1781 auto Load = cast<LoadSDNode>(
Lo);
1783 IncrementPointer(Load, LoVT, MPI, StackPtr);
1785 Hi = DAG.
getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1789 if (LoVT !=
Lo.getValueType())
1791 if (HiVT !=
Hi.getValueType())
1799 assert(
N->getValueType(0).isScalableVector() &&
1800 "Only scalable vectors are supported for STEP_VECTOR");
1808 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1823 Lo = DAG.
getNode(
N->getOpcode(), dl, LoVT,
N->getOperand(0));
1843 EVT MemoryVT =
LD->getMemoryVT();
1847 EVT LoMemVT, HiMemVT;
1854 ReplaceValueWith(
SDValue(LD, 1), NewChain);
1859 LD->getPointerInfo(), LoMemVT,
LD->getOriginalAlign(),
1863 IncrementPointer(LD, LoMemVT, MPI,
Ptr);
1866 HiMemVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
1875 ReplaceValueWith(
SDValue(LD, 1), Ch);
1880 assert(
LD->isUnindexed() &&
"Indexed VP load during type legalization!");
1889 assert(
Offset.isUndef() &&
"Unexpected indexed variable-length load offset");
1890 Align Alignment =
LD->getOriginalAlign();
1893 EVT MemoryVT =
LD->getMemoryVT();
1895 EVT LoMemVT, HiMemVT;
1896 bool HiIsEmpty =
false;
1897 std::tie(LoMemVT, HiMemVT) =
1903 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
1906 GetSplitVector(Mask, MaskLo, MaskHi);
1908 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1913 std::tie(EVLLo, EVLHi) = DAG.
SplitEVL(EVL,
LD->getValueType(0), dl);
1921 MaskLo, EVLLo, LoMemVT, MMO,
LD->isExpandingLoad());
1930 LD->isExpandingLoad());
1936 MPI =
LD->getPointerInfo().getWithOffset(
1941 LD->getAAInfo(),
LD->getRanges());
1944 Offset, MaskHi, EVLHi, HiMemVT, MMO,
1945 LD->isExpandingLoad());
1955 ReplaceValueWith(
SDValue(LD, 1), Ch);
1961 "Indexed VP strided load during type legalization!");
1963 "Unexpected indexed variable-length load offset");
1970 EVT LoMemVT, HiMemVT;
1971 bool HiIsEmpty =
false;
1972 std::tie(LoMemVT, HiMemVT) =
1978 SplitVecRes_SETCC(
Mask.getNode(), LoMask, HiMask);
1981 GetSplitVector(Mask, LoMask, HiMask);
1987 std::tie(LoEVL, HiEVL) =
2025 SLD->
getStride(), HiMask, HiEVL, HiMemVT, MMO,
2036 ReplaceValueWith(
SDValue(SLD, 1), Ch);
2049 assert(
Offset.isUndef() &&
"Unexpected indexed masked load offset");
2058 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
2061 GetSplitVector(Mask, MaskLo, MaskHi);
2063 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
2067 EVT LoMemVT, HiMemVT;
2068 bool HiIsEmpty =
false;
2069 std::tie(LoMemVT, HiMemVT) =
2072 SDValue PassThruLo, PassThruHi;
2074 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2076 std::tie(PassThruLo, PassThruHi) = DAG.
SplitVector(PassThru, dl);
2119 ReplaceValueWith(
SDValue(MLD, 1), Ch);
2136 if (
auto *MSC = dyn_cast<MaskedGatherSDNode>(
N)) {
2137 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2139 auto *VPSC = cast<VPGatherSDNode>(
N);
2140 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2143 EVT MemoryVT =
N->getMemoryVT();
2144 Align Alignment =
N->getOriginalAlign();
2148 if (SplitSETCC && Ops.Mask.getOpcode() ==
ISD::SETCC) {
2149 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2151 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2154 EVT LoMemVT, HiMemVT;
2159 if (getTypeAction(Ops.Index.getValueType()) ==
2161 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2163 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Ops.Index, dl);
2169 if (
auto *MGT = dyn_cast<MaskedGatherSDNode>(
N)) {
2170 SDValue PassThru = MGT->getPassThru();
2171 SDValue PassThruLo, PassThruHi;
2174 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2176 std::tie(PassThruLo, PassThruHi) = DAG.
SplitVector(PassThru, dl);
2181 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo,
Ptr, IndexLo, Ops.Scale};
2183 OpsLo, MMO, IndexTy, ExtType);
2185 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi,
Ptr, IndexHi, Ops.Scale};
2187 OpsHi, MMO, IndexTy, ExtType);
2189 auto *VPGT = cast<VPGatherSDNode>(
N);
2191 std::tie(EVLLo, EVLHi) =
2192 DAG.
SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2194 SDValue OpsLo[] = {Ch,
Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2196 MMO, VPGT->getIndexType());
2198 SDValue OpsHi[] = {Ch,
Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2200 MMO, VPGT->getIndexType());
2210 ReplaceValueWith(
SDValue(
N, 1), Ch);
2214 assert(
N->getValueType(0).isVector() &&
2215 N->getOperand(0).getValueType().isVector() &&
2216 "Operand types must be vectors");
2224 if (getTypeAction(
N->getOperand(0).getValueType()) ==
2226 GetSplitVector(
N->getOperand(0), LL, LH);
2230 if (getTypeAction(
N->getOperand(1).getValueType()) ==
2232 GetSplitVector(
N->getOperand(1), RL, RH);
2237 Lo = DAG.
getNode(
N->getOpcode(),
DL, LoVT, LL, RL,
N->getOperand(2));
2238 Hi = DAG.
getNode(
N->getOpcode(),
DL, HiVT, LH, RH,
N->getOperand(2));
2240 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2241 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2242 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(3));
2243 std::tie(EVLLo, EVLHi) =
2244 DAG.
SplitEVL(
N->getOperand(4),
N->getValueType(0),
DL);
2245 Lo = DAG.
getNode(
N->getOpcode(),
DL, LoVT, LL, RL,
N->getOperand(2), MaskLo,
2247 Hi = DAG.
getNode(
N->getOpcode(),
DL, HiVT, LH, RH,
N->getOperand(2), MaskHi,
2261 EVT InVT =
N->getOperand(0).getValueType();
2263 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
2268 unsigned Opcode =
N->getOpcode();
2269 if (
N->getNumOperands() <= 2) {
2271 Lo = DAG.
getNode(Opcode, dl, LoVT,
Lo,
N->getOperand(1), Flags);
2272 Hi = DAG.
getNode(Opcode, dl, HiVT,
Hi,
N->getOperand(1), Flags);
2280 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
2281 assert(
N->isVPOpcode() &&
"Expected VP opcode");
2284 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
2287 std::tie(EVLLo, EVLHi) =
2288 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
2297 EVT SrcVT =
N->getOperand(0).getValueType();
2298 EVT DestVT =
N->getValueType(0);
2321 EVT SplitLoVT, SplitHiVT;
2325 LLVM_DEBUG(
dbgs() <<
"Split vector extend via incremental extend:";
2326 N->dump(&DAG);
dbgs() <<
"\n");
2327 if (!
N->isVPOpcode()) {
2330 DAG.
getNode(
N->getOpcode(), dl, NewSrcVT,
N->getOperand(0));
2341 DAG.
getNode(
N->getOpcode(), dl, NewSrcVT,
N->getOperand(0),
2342 N->getOperand(1),
N->getOperand(2));
2347 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
2350 std::tie(EVLLo, EVLHi) =
2351 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
2353 Lo = DAG.
getNode(
N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2354 Hi = DAG.
getNode(
N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2359 SplitVecRes_UnaryOp(
N,
Lo,
Hi);
2367 GetSplitVector(
N->getOperand(0), Inputs[0], Inputs[1]);
2368 GetSplitVector(
N->getOperand(1), Inputs[2], Inputs[3]);
2374 return N.getResNo() == 0 &&
2378 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &
DL](
SDValue &Input1,
2383 "Expected build vector node.");
2386 for (
unsigned I = 0;
I < NewElts; ++
I) {
2391 Ops[
I] = Input2.getOperand(
Idx - NewElts);
2393 Ops[
I] = Input1.getOperand(
Idx);
2395 if (Ops[
I].getValueType().bitsGT(EltVT))
2398 return DAG.getBuildVector(NewVT,
DL, Ops);
2406 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT,
this, NewElts,
2410 for (
unsigned Idx = 0;
Idx < std::size(Inputs); ++
Idx) {
2412 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.
getNode());
2421 for (
auto &
P : ShufflesIdxs) {
2422 if (
P.second.size() < 2)
2426 for (
int &
Idx : Mask) {
2429 unsigned SrcRegIdx =
Idx / NewElts;
2430 if (Inputs[SrcRegIdx].
isUndef()) {
2435 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2438 int MaskElt = Shuffle->getMaskElt(
Idx % NewElts);
2443 Idx = MaskElt % NewElts +
2444 P.second[Shuffle->getOperand(MaskElt / NewElts) ==
P.first.first
2450 Inputs[
P.second[0]] =
P.first.first;
2451 Inputs[
P.second[1]] =
P.first.second;
2454 ShufflesIdxs[std::make_pair(
P.first.second,
P.first.first)].clear();
2458 for (
int &
Idx : Mask) {
2461 unsigned SrcRegIdx =
Idx / NewElts;
2462 if (Inputs[SrcRegIdx].
isUndef()) {
2467 getTypeAction(Inputs[SrcRegIdx].getValueType());
2469 Inputs[SrcRegIdx].getNumOperands() == 2 &&
2470 !Inputs[SrcRegIdx].getOperand(1).
isUndef() &&
2473 UsedSubVector.set(2 * SrcRegIdx + (
Idx % NewElts) / (NewElts / 2));
2475 if (UsedSubVector.count() > 1) {
2477 for (
unsigned I = 0;
I < std::size(Inputs); ++
I) {
2478 if (UsedSubVector.test(2 *
I) == UsedSubVector.test(2 *
I + 1))
2480 if (Pairs.
empty() || Pairs.
back().size() == 2)
2482 if (UsedSubVector.test(2 *
I)) {
2483 Pairs.
back().emplace_back(
I, 0);
2485 assert(UsedSubVector.test(2 *
I + 1) &&
2486 "Expected to be used one of the subvectors.");
2487 Pairs.
back().emplace_back(
I, 1);
2490 if (!Pairs.
empty() && Pairs.
front().size() > 1) {
2492 for (
int &
Idx : Mask) {
2495 unsigned SrcRegIdx =
Idx / NewElts;
2497 Pairs, [SrcRegIdx](
ArrayRef<std::pair<unsigned, int>> Idxs) {
2498 return Idxs.front().first == SrcRegIdx ||
2499 Idxs.back().first == SrcRegIdx;
2501 if (It == Pairs.
end())
2503 Idx = It->front().first * NewElts + (
Idx % NewElts) % (NewElts / 2) +
2504 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2507 for (
ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2508 Inputs[Idxs.front().first] = DAG.
getNode(
2510 Inputs[Idxs.front().first].getValueType(),
2511 Inputs[Idxs.front().first].
getOperand(Idxs.front().second),
2512 Inputs[Idxs.back().first].
getOperand(Idxs.back().second));
2521 for (
unsigned I = 0;
I < std::size(Inputs); ++
I) {
2522 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[
I].getNode());
2525 if (Shuffle->getOperand(0).getValueType() != NewVT)
2528 if (!Inputs[
I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2529 !Shuffle->isSplat()) {
2531 }
else if (!Inputs[
I].hasOneUse() &&
2532 !Shuffle->getOperand(1).isUndef()) {
2534 for (
int &
Idx : Mask) {
2537 unsigned SrcRegIdx =
Idx / NewElts;
2540 int MaskElt = Shuffle->getMaskElt(
Idx % NewElts);
2545 int OpIdx = MaskElt / NewElts;
2558 for (
int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2559 if (Shuffle->getOperand(OpIdx).isUndef())
2561 auto *It =
find(Inputs, Shuffle->getOperand(OpIdx));
2562 if (It == std::end(Inputs))
2564 int FoundOp = std::distance(std::begin(Inputs), It);
2567 for (
int &
Idx : Mask) {
2570 unsigned SrcRegIdx =
Idx / NewElts;
2573 int MaskElt = Shuffle->getMaskElt(
Idx % NewElts);
2578 int MaskIdx = MaskElt / NewElts;
2579 if (OpIdx == MaskIdx)
2580 Idx = MaskElt % NewElts + FoundOp * NewElts;
2583 Op = (OpIdx + 1) % 2;
2591 for (
int &
Idx : Mask) {
2594 unsigned SrcRegIdx =
Idx / NewElts;
2597 int MaskElt = Shuffle->getMaskElt(
Idx % NewElts);
2598 int OpIdx = MaskElt / NewElts;
2601 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2607 TryPeekThroughShufflesInputs(OrigMask);
2609 auto &&MakeUniqueInputs = [&Inputs, &
IsConstant,
2613 for (
const auto &
I : Inputs) {
2615 UniqueConstantInputs.
insert(
I);
2616 else if (!
I.isUndef())
2621 if (UniqueInputs.
size() != std::size(Inputs)) {
2622 auto &&UniqueVec = UniqueInputs.
takeVector();
2623 auto &&UniqueConstantVec = UniqueConstantInputs.
takeVector();
2624 unsigned ConstNum = UniqueConstantVec.size();
2625 for (
int &
Idx : Mask) {
2628 unsigned SrcRegIdx =
Idx / NewElts;
2629 if (Inputs[SrcRegIdx].
isUndef()) {
2633 const auto It =
find(UniqueConstantVec, Inputs[SrcRegIdx]);
2634 if (It != UniqueConstantVec.end()) {
2636 NewElts * std::distance(UniqueConstantVec.begin(), It);
2637 assert(
Idx >= 0 &&
"Expected defined mask idx.");
2640 const auto RegIt =
find(UniqueVec, Inputs[SrcRegIdx]);
2641 assert(RegIt != UniqueVec.end() &&
"Cannot find non-const value.");
2643 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2644 assert(
Idx >= 0 &&
"Expected defined mask idx.");
2646 copy(UniqueConstantVec, std::begin(Inputs));
2647 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2650 MakeUniqueInputs(OrigMask);
2652 copy(Inputs, std::begin(OrigInputs));
2658 unsigned FirstMaskIdx =
High * NewElts;
2661 assert(!Output &&
"Expected default initialized initial value.");
2662 TryPeekThroughShufflesInputs(Mask);
2663 MakeUniqueInputs(Mask);
2665 copy(Inputs, std::begin(TmpInputs));
2668 bool SecondIteration =
false;
2669 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](
unsigned Idx) {
2674 if (UsedIdx >= 0 &&
static_cast<unsigned>(UsedIdx) ==
Idx)
2675 SecondIteration =
true;
2676 return SecondIteration;
2679 Mask, std::size(Inputs), std::size(Inputs),
2681 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2682 [&Output, &DAG = DAG, NewVT, &
DL, &Inputs,
2685 Output = BuildVector(Inputs[
Idx], Inputs[
Idx], Mask);
2687 Output = DAG.getVectorShuffle(NewVT,
DL, Inputs[
Idx],
2688 DAG.getUNDEF(NewVT), Mask);
2689 Inputs[
Idx] = Output;
2691 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &
DL, &Inputs,
2694 if (AccumulateResults(Idx1)) {
2697 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2699 Output = DAG.getVectorShuffle(NewVT,
DL, Inputs[Idx1],
2700 Inputs[Idx2], Mask);
2704 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2706 Output = DAG.getVectorShuffle(NewVT,
DL, TmpInputs[Idx1],
2707 TmpInputs[Idx2], Mask);
2709 Inputs[Idx1] = Output;
2711 copy(OrigInputs, std::begin(Inputs));
2716 EVT OVT =
N->getValueType(0);
2723 const Align Alignment =
2724 DAG.getDataLayout().getABITypeAlign(NVT.
getTypeForEVT(*DAG.getContext()));
2726 Lo = DAG.getVAArg(NVT, dl, Chain,
Ptr, SV, Alignment.
value());
2727 Hi = DAG.getVAArg(NVT, dl,
Lo.getValue(1),
Ptr, SV, Alignment.
value());
2728 Chain =
Hi.getValue(1);
2732 ReplaceValueWith(
SDValue(
N, 1), Chain);
2737 EVT DstVTLo, DstVTHi;
2738 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(
N->getValueType(0));
2742 EVT SrcVT =
N->getOperand(0).getValueType();
2744 GetSplitVector(
N->getOperand(0), SrcLo, SrcHi);
2746 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(
N, 0);
2748 Lo = DAG.getNode(
N->getOpcode(), dl, DstVTLo, SrcLo,
N->getOperand(1));
2749 Hi = DAG.getNode(
N->getOpcode(), dl, DstVTHi, SrcHi,
N->getOperand(1));
2755 GetSplitVector(
N->getOperand(0), InLo, InHi);
2764 EVT VT =
N->getValueType(0);
2768 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2772 DAG.getVectorIdxConstant(0,
DL));
2778void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(
SDNode *
N) {
2780 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2781 GetSplitVector(
N->getOperand(0), Op0Lo, Op0Hi);
2782 GetSplitVector(
N->getOperand(1), Op1Lo, Op1Hi);
2786 DAG.getVTList(VT, VT), Op0Lo, Op0Hi);
2788 DAG.getVTList(VT, VT), Op1Lo, Op1Hi);
2794void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(
SDNode *
N) {
2795 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2796 GetSplitVector(
N->getOperand(0), Op0Lo, Op0Hi);
2797 GetSplitVector(
N->getOperand(1), Op1Lo, Op1Hi);
2801 DAG.getVTList(VT, VT), Op0Lo, Op1Lo),
2803 DAG.getVTList(VT, VT), Op0Hi, Op1Hi)};
2805 SetSplitVector(
SDValue(
N, 0), Res[0].getValue(0), Res[0].getValue(1));
2806 SetSplitVector(
SDValue(
N, 1), Res[1].getValue(0), Res[1].getValue(1));
2817bool DAGTypeLegalizer::SplitVectorOperand(
SDNode *
N,
unsigned OpNo) {
2822 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
2825 switch (
N->getOpcode()) {
2828 dbgs() <<
"SplitVectorOperand Op #" << OpNo <<
": ";
2836 case ISD::SETCC: Res = SplitVecOp_VSETCC(
N);
break;
2842 case ISD::VP_TRUNCATE:
2844 Res = SplitVecOp_TruncateHelper(
N);
2847 case ISD::VP_FP_ROUND:
2851 Res = SplitVecOp_STORE(cast<StoreSDNode>(
N), OpNo);
2854 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(
N), OpNo);
2856 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2857 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(
N), OpNo);
2860 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(
N), OpNo);
2863 case ISD::VP_SCATTER:
2864 Res = SplitVecOp_Scatter(cast<MemSDNode>(
N), OpNo);
2867 case ISD::VP_GATHER:
2868 Res = SplitVecOp_Gather(cast<MemSDNode>(
N), OpNo);
2871 Res = SplitVecOp_VSELECT(
N, OpNo);
2877 case ISD::VP_SINT_TO_FP:
2878 case ISD::VP_UINT_TO_FP:
2879 if (
N->getValueType(0).bitsLT(
2880 N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2881 Res = SplitVecOp_TruncateHelper(
N);
2883 Res = SplitVecOp_UnaryOp(
N);
2887 Res = SplitVecOp_FP_TO_XINT_SAT(
N);
2891 case ISD::VP_FP_TO_SINT:
2892 case ISD::VP_FP_TO_UINT:
2901 Res = SplitVecOp_UnaryOp(
N);
2907 Res = SplitVecOp_ExtVecInRegOp(
N);
2923 Res = SplitVecOp_VECREDUCE(
N, OpNo);
2927 Res = SplitVecOp_VECREDUCE_SEQ(
N);
2929 case ISD::VP_REDUCE_FADD:
2930 case ISD::VP_REDUCE_SEQ_FADD:
2931 case ISD::VP_REDUCE_FMUL:
2932 case ISD::VP_REDUCE_SEQ_FMUL:
2933 case ISD::VP_REDUCE_ADD:
2934 case ISD::VP_REDUCE_MUL:
2935 case ISD::VP_REDUCE_AND:
2936 case ISD::VP_REDUCE_OR:
2937 case ISD::VP_REDUCE_XOR:
2938 case ISD::VP_REDUCE_SMAX:
2939 case ISD::VP_REDUCE_SMIN:
2940 case ISD::VP_REDUCE_UMAX:
2941 case ISD::VP_REDUCE_UMIN:
2942 case ISD::VP_REDUCE_FMAX:
2943 case ISD::VP_REDUCE_FMIN:
2944 Res = SplitVecOp_VP_REDUCE(
N, OpNo);
2949 if (!Res.getNode())
return false;
2953 if (Res.getNode() ==
N)
2956 if (
N->isStrictFPOpcode())
2957 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 2 &&
2958 "Invalid operand expansion");
2960 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 1 &&
2961 "Invalid operand expansion");
2963 ReplaceValueWith(
SDValue(
N, 0), Res);
2967SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(
SDNode *
N,
unsigned OpNo) {
2970 assert(OpNo == 0 &&
"Illegal operand must be mask");
2977 assert(
Mask.getValueType().isVector() &&
"VSELECT without a vector mask?");
2980 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
2981 assert(
Lo.getValueType() ==
Hi.getValueType() &&
2982 "Lo and Hi have differing types");
2985 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2986 assert(LoOpVT == HiOpVT &&
"Asymmetric vector split?");
2988 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2989 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0,
DL);
2990 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1,
DL);
2991 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask,
DL);
3001SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(
SDNode *
N,
unsigned OpNo) {
3002 EVT ResVT =
N->getValueType(0);
3006 SDValue VecOp =
N->getOperand(OpNo);
3008 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
3009 GetSplitVector(VecOp,
Lo,
Hi);
3011 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3017 return DAG.getNode(
N->getOpcode(), dl, ResVT, Partial,
N->getFlags());
3021 EVT ResVT =
N->getValueType(0);
3030 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
3031 GetSplitVector(VecOp,
Lo,
Hi);
3033 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3039 return DAG.getNode(
N->getOpcode(), dl, ResVT, Partial,
Hi, Flags);
3042SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
3043 assert(
N->isVPOpcode() &&
"Expected VP opcode");
3044 assert(OpNo == 1 &&
"Can only split reduce vector operand");
3046 unsigned Opc =
N->getOpcode();
3047 EVT ResVT =
N->getValueType(0);
3051 SDValue VecOp =
N->getOperand(OpNo);
3053 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
3054 GetSplitVector(VecOp,
Lo,
Hi);
3057 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(2));
3060 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(
N->getOperand(3), VecVT, dl);
3065 DAG.
getNode(Opc, dl, ResVT, {
N->getOperand(0),
Lo, MaskLo, EVLLo},
Flags);
3066 return DAG.getNode(Opc, dl, ResVT, {ResLo,
Hi, MaskHi, EVLHi},
Flags);
3071 EVT ResVT =
N->getValueType(0);
3074 GetSplitVector(
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0),
Lo,
Hi);
3075 EVT InVT =
Lo.getValueType();
3080 if (
N->isStrictFPOpcode()) {
3081 Lo = DAG.getNode(
N->getOpcode(), dl, { OutVT, MVT::Other },
3082 { N->getOperand(0), Lo });
3083 Hi = DAG.getNode(
N->getOpcode(), dl, { OutVT, MVT::Other },
3084 { N->getOperand(0), Hi });
3093 ReplaceValueWith(
SDValue(
N, 1), Ch);
3094 }
else if (
N->getNumOperands() == 3) {
3095 assert(
N->isVPOpcode() &&
"Expected VP opcode");
3096 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3097 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
3098 std::tie(EVLLo, EVLHi) =
3099 DAG.SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
3100 Lo = DAG.getNode(
N->getOpcode(), dl, OutVT,
Lo, MaskLo, EVLLo);
3101 Hi = DAG.getNode(
N->getOpcode(), dl, OutVT,
Hi, MaskHi, EVLHi);
3103 Lo = DAG.getNode(
N->getOpcode(), dl, OutVT,
Lo);
3104 Hi = DAG.getNode(
N->getOpcode(), dl, OutVT,
Hi);
3115 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3116 Lo = BitConvertToInteger(
Lo);
3117 Hi = BitConvertToInteger(
Hi);
3119 if (DAG.getDataLayout().isBigEndian())
3123 JoinIntegers(
Lo,
Hi));
3128 assert(OpNo == 1 &&
"Invalid OpNo; can only split SubVec.");
3130 EVT ResVT =
N->getValueType(0);
3138 GetSplitVector(SubVec,
Lo,
Hi);
3140 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
3141 uint64_t LoElts =
Lo.getValueType().getVectorMinNumElements();
3147 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3149 return SecondInsertion;
3152SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
3154 EVT SubVT =
N->getValueType(0);
3159 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3161 uint64_t LoEltsMin =
Lo.getValueType().getVectorMinNumElements();
3162 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
3164 if (IdxVal < LoEltsMin) {
3166 "Extracted subvector crosses vector split!");
3169 N->getOperand(0).getValueType().isScalableVector())
3171 DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3176 "Extracting scalable subvector from fixed-width unsupported");
3184 "subvector from a scalable predicate vector");
3190 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
3192 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
3193 auto &MF = DAG.getMachineFunction();
3197 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3204 SubVT, dl, Store, StackPtr,
3208SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
3217 GetSplitVector(Vec,
Lo,
Hi);
3219 uint64_t LoElts =
Lo.getValueType().getVectorMinNumElements();
3221 if (IdxVal < LoElts)
3225 DAG.getConstant(IdxVal - LoElts,
SDLoc(
N),
3226 Idx.getValueType())), 0);
3230 if (CustomLowerNode(
N,
N->getValueType(0),
true))
3246 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
3248 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
3249 auto &MF = DAG.getMachineFunction();
3252 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3260 if (
N->getValueType(0).bitsLT(EltVT)) {
3261 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3263 return DAG.getZExtOrTrunc(Load, dl,
N->getValueType(0));
3266 return DAG.getExtLoad(
3277 SplitVecRes_ExtVecInRegOp(
N,
Lo,
Hi);
3285 SplitVecRes_Gather(
N,
Lo,
Hi);
3288 ReplaceValueWith(
SDValue(
N, 0), Res);
3293 assert(
N->isUnindexed() &&
"Indexed vp_store of vector?");
3297 assert(
Offset.isUndef() &&
"Unexpected VP store offset");
3299 SDValue EVL =
N->getVectorLength();
3301 Align Alignment =
N->getOriginalAlign();
3307 GetSplitVector(
Data, DataLo, DataHi);
3309 std::tie(DataLo, DataHi) = DAG.SplitVector(
Data,
DL);
3314 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
3317 GetSplitVector(Mask, MaskLo, MaskHi);
3319 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask,
DL);
3322 EVT MemoryVT =
N->getMemoryVT();
3323 EVT LoMemVT, HiMemVT;
3324 bool HiIsEmpty =
false;
3325 std::tie(LoMemVT, HiMemVT) =
3326 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.
getValueType(), &HiIsEmpty);
3330 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL,
Data.getValueType(),
DL);
3337 Lo = DAG.getStoreVP(Ch,
DL, DataLo,
Ptr,
Offset, MaskLo, EVLLo, LoMemVT, MMO,
3338 N->getAddressingMode(),
N->isTruncatingStore(),
3339 N->isCompressingStore());
3346 N->isCompressingStore());
3354 MPI =
N->getPointerInfo().getWithOffset(
3357 MMO = DAG.getMachineFunction().getMachineMemOperand(
3359 N->getAAInfo(),
N->getRanges());
3361 Hi = DAG.getStoreVP(Ch,
DL, DataHi,
Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
3362 N->getAddressingMode(),
N->isTruncatingStore(),
3363 N->isCompressingStore());
3372 assert(
N->isUnindexed() &&
"Indexed vp_strided_store of a vector?");
3373 assert(
N->getOffset().isUndef() &&
"Unexpected VP strided store offset");
3380 GetSplitVector(
Data, LoData, HiData);
3382 std::tie(LoData, HiData) = DAG.SplitVector(
Data,
DL);
3384 EVT LoMemVT, HiMemVT;
3385 bool HiIsEmpty =
false;
3386 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3392 SplitVecRes_SETCC(
Mask.getNode(), LoMask, HiMask);
3393 else if (getTypeAction(
Mask.getValueType()) ==
3395 GetSplitVector(Mask, LoMask, HiMask);
3397 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask,
DL);
3400 std::tie(LoEVL, HiEVL) =
3401 DAG.SplitEVL(
N->getVectorLength(),
Data.getValueType(),
DL);
3405 N->getChain(),
DL, LoData,
N->getBasePtr(),
N->getOffset(),
3406 N->getStride(), LoMask, LoEVL, LoMemVT,
N->getMemOperand(),
3407 N->getAddressingMode(),
N->isTruncatingStore(),
N->isCompressingStore());
3418 EVT PtrVT =
N->getBasePtr().getValueType();
3421 DAG.getSExtOrTrunc(
N->getStride(),
DL, PtrVT));
3424 Align Alignment =
N->getOriginalAlign();
3432 N->getAAInfo(),
N->getRanges());
3435 N->getChain(),
DL, HiData,
Ptr,
N->getOffset(),
N->getStride(), HiMask,
3436 HiEVL, HiMemVT, MMO,
N->getAddressingMode(),
N->isTruncatingStore(),
3437 N->isCompressingStore());
3446 assert(
N->isUnindexed() &&
"Indexed masked store of vector?");
3450 assert(
Offset.isUndef() &&
"Unexpected indexed masked store offset");
3453 Align Alignment =
N->getOriginalAlign();
3459 GetSplitVector(
Data, DataLo, DataHi);
3461 std::tie(DataLo, DataHi) = DAG.SplitVector(
Data,
DL);
3466 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
3469 GetSplitVector(Mask, MaskLo, MaskHi);
3471 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask,
DL);
3474 EVT MemoryVT =
N->getMemoryVT();
3475 EVT LoMemVT, HiMemVT;
3476 bool HiIsEmpty =
false;
3477 std::tie(LoMemVT, HiMemVT) =
3478 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.
getValueType(), &HiIsEmpty);
3485 Lo = DAG.getMaskedStore(Ch,
DL, DataLo,
Ptr,
Offset, MaskLo, LoMemVT, MMO,
3486 N->getAddressingMode(),
N->isTruncatingStore(),
3487 N->isCompressingStore());
3496 N->isCompressingStore());
3504 MPI =
N->getPointerInfo().getWithOffset(
3507 MMO = DAG.getMachineFunction().getMachineMemOperand(
3509 N->getAAInfo(),
N->getRanges());
3511 Hi = DAG.getMaskedStore(Ch,
DL, DataHi,
Ptr,
Offset, MaskHi, HiMemVT, MMO,
3512 N->getAddressingMode(),
N->isTruncatingStore(),
3513 N->isCompressingStore());
3526 EVT MemoryVT =
N->getMemoryVT();
3527 Align Alignment =
N->getOriginalAlign();
3535 if (
auto *MSC = dyn_cast<MaskedScatterSDNode>(
N)) {
3536 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3539 auto *VPSC = cast<VPScatterSDNode>(
N);
3540 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3545 EVT LoMemVT, HiMemVT;
3546 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3551 GetSplitVector(Ops.Data, DataLo, DataHi);
3553 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data,
DL);
3557 if (OpNo == 1 && Ops.Mask.getOpcode() ==
ISD::SETCC) {
3558 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3560 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask,
DL);
3564 if (getTypeAction(Ops.Index.getValueType()) ==
3566 GetSplitVector(Ops.Index, IndexLo, IndexHi);
3568 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index,
DL);
3575 if (
auto *MSC = dyn_cast<MaskedScatterSDNode>(
N)) {
3576 SDValue OpsLo[] = {Ch, DataLo, MaskLo,
Ptr, IndexLo, Ops.Scale};
3578 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
DL, OpsLo, MMO,
3579 MSC->getIndexType(), MSC->isTruncatingStore());
3584 SDValue OpsHi[] = {
Lo, DataHi, MaskHi,
Ptr, IndexHi, Ops.Scale};
3585 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
DL, OpsHi,
3586 MMO, MSC->getIndexType(),
3587 MSC->isTruncatingStore());
3589 auto *VPSC = cast<VPScatterSDNode>(
N);
3591 std::tie(EVLLo, EVLHi) =
3592 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(),
DL);
3594 SDValue OpsLo[] = {Ch, DataLo,
Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3595 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT,
DL, OpsLo, MMO,
3596 VPSC->getIndexType());
3601 SDValue OpsHi[] = {
Lo, DataHi,
Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3602 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT,
DL, OpsHi, MMO,
3603 VPSC->getIndexType());
3607 assert(
N->isUnindexed() &&
"Indexed store of vector?");
3608 assert(OpNo == 1 &&
"Can only split the stored value");
3611 bool isTruncating =
N->isTruncatingStore();
3614 EVT MemoryVT =
N->getMemoryVT();
3615 Align Alignment =
N->getOriginalAlign();
3619 GetSplitVector(
N->getOperand(1),
Lo,
Hi);
3621 EVT LoMemVT, HiMemVT;
3622 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3629 Lo = DAG.getTruncStore(Ch,
DL,
Lo,
Ptr,
N->getPointerInfo(), LoMemVT,
3630 Alignment, MMOFlags, AAInfo);
3632 Lo = DAG.getStore(Ch,
DL,
Lo,
Ptr,
N->getPointerInfo(), Alignment, MMOFlags,
3636 IncrementPointer(
N, LoMemVT, MPI,
Ptr);
3639 Hi = DAG.getTruncStore(Ch,
DL,
Hi,
Ptr, MPI,
3640 HiMemVT, Alignment, MMOFlags, AAInfo);
3642 Hi = DAG.getStore(Ch,
DL,
Hi,
Ptr, MPI, Alignment, MMOFlags, AAInfo);
3656 EVT EltVT =
N->getValueType(0).getVectorElementType();
3657 for (
const SDValue &Op :
N->op_values()) {
3658 for (
unsigned i = 0, e =
Op.getValueType().getVectorNumElements();
3661 DAG.getVectorIdxConstant(i,
DL)));
3665 return DAG.getBuildVector(
N->getValueType(0),
DL, Elts);
3686 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
3687 SDValue InVec =
N->getOperand(OpNo);
3689 EVT OutVT =
N->getValueType(0);
3697 EVT LoOutVT, HiOutVT;
3698 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3699 assert(LoOutVT == HiOutVT &&
"Unequal split?");
3704 if (isTypeLegal(LoOutVT) ||
3705 InElementSize <= OutElementSize * 2)
3706 return SplitVecOp_UnaryOp(
N);
3715 return SplitVecOp_UnaryOp(
N);
3719 GetSplitVector(InVec, InLoVec, InHiVec);
3725 EVT HalfElementVT = IsFloat ?
3727 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3734 if (
N->isStrictFPOpcode()) {
3735 HalfLo = DAG.
getNode(
N->getOpcode(),
DL, {HalfVT, MVT::Other},
3736 {N->getOperand(0), InLoVec});
3737 HalfHi = DAG.
getNode(
N->getOpcode(),
DL, {HalfVT, MVT::Other},
3738 {N->getOperand(0), InHiVec});
3744 HalfLo = DAG.
getNode(
N->getOpcode(),
DL, HalfVT, InLoVec);
3745 HalfHi = DAG.
getNode(
N->getOpcode(),
DL, HalfVT, InHiVec);
3757 if (
N->isStrictFPOpcode()) {
3761 DAG.getTargetConstant(0,
DL, TLI.
getPointerTy(DAG.getDataLayout()))});
3769 DAG.getTargetConstant(
3775 assert(
N->getValueType(0).isVector() &&
3776 N->getOperand(0).getValueType().isVector() &&
3777 "Operand types must be vectors");
3779 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3781 GetSplitVector(
N->getOperand(0), Lo0, Hi0);
3782 GetSplitVector(
N->getOperand(1), Lo1, Hi1);
3793 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
3794 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3795 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(3));
3796 std::tie(EVLLo, EVLHi) =
3797 DAG.SplitEVL(
N->getOperand(4),
N->getValueType(0),
DL);
3798 LoRes = DAG.
getNode(ISD::VP_SETCC,
DL, PartResVT, Lo0, Lo1,
3799 N->getOperand(2), MaskLo, EVLLo);
3800 HiRes = DAG.
getNode(ISD::VP_SETCC,
DL, PartResVT, Hi0, Hi1,
3801 N->getOperand(2), MaskHi, EVLHi);
3805 EVT OpVT =
N->getOperand(0).getValueType();
3808 return DAG.getNode(ExtendCode,
DL,
N->getValueType(0), Con);
3814 EVT ResVT =
N->getValueType(0);
3817 GetSplitVector(
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0),
Lo,
Hi);
3818 EVT InVT =
Lo.getValueType();
3823 if (
N->isStrictFPOpcode()) {
3824 Lo = DAG.getNode(
N->getOpcode(),
DL, { OutVT, MVT::Other },
3825 { N->getOperand(0), Lo, N->getOperand(2) });
3826 Hi = DAG.getNode(
N->getOpcode(),
DL, { OutVT, MVT::Other },
3827 { N->getOperand(0), Hi, N->getOperand(2) });
3831 Lo.getValue(1),
Hi.getValue(1));
3832 ReplaceValueWith(
SDValue(
N, 1), NewChain);
3833 }
else if (
N->getOpcode() == ISD::VP_FP_ROUND) {
3834 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3835 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
3836 std::tie(EVLLo, EVLHi) =
3837 DAG.SplitEVL(
N->getOperand(2),
N->getValueType(0),
DL);
3838 Lo = DAG.getNode(ISD::VP_FP_ROUND,
DL, OutVT,
Lo, MaskLo, EVLLo);
3839 Hi = DAG.getNode(ISD::VP_FP_ROUND,
DL, OutVT,
Hi, MaskHi, EVLHi);
3854 EVT LHSLoVT, LHSHiVT;
3855 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
3857 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
3858 return DAG.UnrollVectorOp(
N,
N->getValueType(0).getVectorNumElements());
3861 std::tie(LHSLo, LHSHi) =
3862 DAG.SplitVector(
N->getOperand(0),
DL, LHSLoVT, LHSHiVT);
3865 std::tie(RHSLo, RHSHi) = DAG.SplitVector(
N->getOperand(1),
DL);
3874 EVT ResVT =
N->getValueType(0);
3877 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3878 EVT InVT =
Lo.getValueType();
3884 Lo = DAG.getNode(
N->getOpcode(), dl, NewResVT,
Lo,
N->getOperand(1));
3885 Hi = DAG.getNode(
N->getOpcode(), dl, NewResVT,
Hi,
N->getOperand(1));
3894void DAGTypeLegalizer::WidenVectorResult(
SDNode *
N,
unsigned ResNo) {
3895 LLVM_DEBUG(
dbgs() <<
"Widen node result " << ResNo <<
": ";
N->dump(&DAG);
3899 if (CustomWidenLowerNode(
N,
N->getValueType(ResNo)))
3904 auto unrollExpandedOp = [&]() {
3909 EVT VT =
N->getValueType(0);
3919 switch (
N->getOpcode()) {
3922 dbgs() <<
"WidenVectorResult #" << ResNo <<
": ";
3934 Res = WidenVecRes_INSERT_SUBVECTOR(
N);
3938 case ISD::LOAD: Res = WidenVecRes_LOAD(
N);
break;
3942 Res = WidenVecRes_ScalarOp(
N);
3947 case ISD::VP_SELECT:
3949 Res = WidenVecRes_Select(
N);
3953 case ISD::SETCC: Res = WidenVecRes_SETCC(
N);
break;
3954 case ISD::UNDEF: Res = WidenVecRes_UNDEF(
N);
break;
3956 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(
N));
3959 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(
N));
3961 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
3962 Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(
N));
3965 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(
N));
3968 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(
N));
3970 case ISD::VP_GATHER:
3971 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(
N));
3974 Res = WidenVecRes_VECTOR_REVERSE(
N);
3982 case ISD::OR:
case ISD::VP_OR:
4022 case ISD::VP_FCOPYSIGN:
4023 Res = WidenVecRes_Binary(
N);
4028 if (unrollExpandedOp())
4043 Res = WidenVecRes_BinaryCanTrap(
N);
4052 Res = WidenVecRes_BinaryWithExtraScalarOp(
N);
4055#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4056 case ISD::STRICT_##DAGN:
4057#include "llvm/IR/ConstrainedOps.def"
4058 Res = WidenVecRes_StrictFP(
N);
4067 Res = WidenVecRes_OverflowOp(
N, ResNo);
4071 Res = WidenVecRes_FCOPYSIGN(
N);
4075 Res = WidenVecRes_IS_FPCLASS(
N);
4079 Res = WidenVecRes_POWI(
N);
4085 Res = WidenVecRes_EXTEND_VECTOR_INREG(
N);
4090 case ISD::VP_FP_EXTEND:
4092 case ISD::VP_FP_ROUND:
4094 case ISD::VP_FP_TO_SINT:
4096 case ISD::VP_FP_TO_UINT:
4098 case ISD::VP_SIGN_EXTEND:
4100 case ISD::VP_SINT_TO_FP:
4101 case ISD::VP_TRUNCATE:
4104 case ISD::VP_UINT_TO_FP:
4106 case ISD::VP_ZERO_EXTEND:
4107 Res = WidenVecRes_Convert(
N);
4112 Res = WidenVecRes_FP_TO_XINT_SAT(
N);
4131 if (unrollExpandedOp())
4141 case ISD::VP_BITREVERSE:
4147 case ISD::VP_CTLZ_ZERO_UNDEF: