35#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::ScalarizeVectorResult(
SDNode *
N,
unsigned ResNo) {
47 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
50 switch (
N->getOpcode()) {
53 dbgs() <<
"ScalarizeVectorResult #" << ResNo <<
": ";
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(
N);
68 R = ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(
N);
72 R = ScalarizeVecRes_VECTOR_SHUFFLE_VAR(
N);
76 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(
N);
79 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(
N);
85 R = ScalarizeVecRes_UnaryOpWithExtraInput(
N);
120 R = ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
126 case ISD::SETCC: R = ScalarizeVecRes_SETCC(
N);
break;
128 R = ScalarizeVecRes_VECTOR_MATCH(
N);
131 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(
N);
break;
137 R = ScalarizeVecRes_VecInregOp(
N);
189 R = ScalarizeVecRes_UnaryOp(
N);
192 R = ScalarizeVecRes_ADDRSPACECAST(
N);
198 R = ScalarizeVecRes_UnaryOpWithTwoResults(
N, ResNo);
256 R = ScalarizeVecRes_BinOp(
N);
263 R = ScalarizeVecRes_MaskedBinOp(
N);
267 R = ScalarizeVecRes_FPOp_MultiType(
N);
272 R = ScalarizeVecRes_CMP(
N);
278 R = ScalarizeVecRes_TernaryOp(
N);
281#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
282 case ISD::STRICT_##DAGN:
283#include "llvm/IR/ConstrainedOps.def"
284 R = ScalarizeVecRes_StrictFPOp(
N);
289 R = ScalarizeVecRes_FP_TO_XINT_SAT(
N);
298 R = ScalarizeVecRes_OverflowOp(
N, ResNo);
308 R = ScalarizeVecRes_FIX(
N);
314 R = ScalarizeVecRes_PartialReduction(
N);
320 SetScalarizedVector(
SDValue(
N, ResNo), R);
324 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
325 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
326 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
332 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
333 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
335 EVT MaskVT =
Mask.getValueType();
340 Mask = GetScalarizedVector(Mask);
349 DAG.getConstant(1,
DL,
LHS.getValueType()));
351 LHS.getValueType(),
LHS, Divisor);
359 if (getTypeAction(
LHS.getValueType()) ==
361 LHS = GetScalarizedVector(
LHS);
362 RHS = GetScalarizedVector(
RHS);
364 EVT VT =
LHS.getValueType().getVectorElementType();
365 LHS = DAG.getExtractVectorElt(
DL, VT,
LHS, 0);
366 RHS = DAG.getExtractVectorElt(
DL, VT,
RHS, 0);
369 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
370 N->getValueType(0).getVectorElementType(),
LHS,
RHS);
374 SDValue Op0 = GetScalarizedVector(
N->getOperand(0));
375 SDValue Op1 = GetScalarizedVector(
N->getOperand(1));
376 SDValue Op2 = GetScalarizedVector(
N->getOperand(2));
377 return DAG.getNode(
N->getOpcode(), SDLoc(
N), Op0.
getValueType(), Op0, Op1,
382 SDValue Op0 = GetScalarizedVector(
N->getOperand(0));
383 SDValue Op1 = GetScalarizedVector(
N->getOperand(1));
389SDValue DAGTypeLegalizer::ScalarizeVecRes_PartialReduction(
SDNode *
N) {
390 SDValue Acc = GetScalarizedVector(
N->getOperand(0));
394 EVT InVT =
L.getValueType();
409DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(
SDNode *
N,
411 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
412 "Unexpected vector type!");
413 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
415 EVT VT0 =
N->getValueType(0);
416 EVT VT1 =
N->getValueType(1);
420 DAG.getNode(
N->getOpcode(), dl,
421 {VT0.getScalarType(), VT1.getScalarType()}, Elt)
425 unsigned OtherNo = 1 - ResNo;
426 EVT OtherVT =
N->getValueType(OtherNo);
428 SetScalarizedVector(
SDValue(
N, OtherNo),
SDValue(ScalarNode, OtherNo));
432 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
435 return SDValue(ScalarNode, ResNo);
440 unsigned NumOpers =
N->getNumOperands();
442 EVT ValueVTs[] = {VT, MVT::Other};
451 for (
unsigned i = 1; i < NumOpers; ++i) {
457 Oper = GetScalarizedVector(Oper);
466 SDValue Result = DAG.getNode(
N->getOpcode(), dl, DAG.getVTList(ValueVTs),
467 Opers,
N->getFlags());
478 EVT ResVT =
N->getValueType(0);
479 EVT OvVT =
N->getValueType(1);
483 ScalarLHS = GetScalarizedVector(
N->getOperand(0));
484 ScalarRHS = GetScalarizedVector(
N->getOperand(1));
487 DAG.ExtractVectorElements(
N->getOperand(0), ElemsLHS);
488 DAG.ExtractVectorElements(
N->getOperand(1), ElemsRHS);
489 ScalarLHS = ElemsLHS[0];
490 ScalarRHS = ElemsRHS[0];
493 SDVTList ScalarVTs = DAG.getVTList(
495 SDNode *ScalarNode = DAG.getNode(
N->getOpcode(),
DL, ScalarVTs,
496 {ScalarLHS, ScalarRHS},
N->getFlags())
500 unsigned OtherNo = 1 - ResNo;
501 EVT OtherVT =
N->getValueType(OtherNo);
503 SetScalarizedVector(
SDValue(
N, OtherNo),
SDValue(ScalarNode, OtherNo));
507 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
510 return SDValue(ScalarNode, ResNo);
515 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
516 return GetScalarizedVector(
Op);
519SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(
SDNode *
N) {
524 N->getValueType(0).getScalarType(), Mask,
525 DAG.getVectorIdxConstant(0,
DL));
531 Op = GetScalarizedVector(
Op);
532 EVT NewVT =
N->getValueType(0).getVectorElementType();
537SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(
SDNode *
N) {
547SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
549 N->getValueType(0).getVectorElementType(),
550 N->getOperand(0),
N->getOperand(1));
553SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE_VAR(
SDNode *
N) {
555 return GetScalarizedVector(
N->getOperand(0));
561 EVT OpVT =
Op.getValueType();
565 Op = GetScalarizedVector(
Op);
568 Op = DAG.getExtractVectorElt(
DL, VT,
Op, 0);
571 N->getValueType(0).getVectorElementType(),
Op,
575SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(
SDNode *
N) {
578 EVT OpVT =
Op.getValueType();
582 Op = GetScalarizedVector(
Op);
585 Op = DAG.getExtractVectorElt(
DL, VT,
Op, 0);
588 N->getValueType(0).getVectorElementType(),
Op,
592SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(
SDNode *
N) {
595 EVT OpVT =
Op.getValueType();
598 Op = GetScalarizedVector(
Op);
601 Op = DAG.getExtractVectorElt(
DL, VT,
Op, 0);
604 N->getValueType(0).getVectorElementType(),
Op,
605 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
608SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(
SDNode *
N) {
609 SDValue Op = GetScalarizedVector(
N->getOperand(0));
610 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
Op.getValueType(),
Op,
614SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOp_MultiType(
SDNode *
N) {
616 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
618 EVT RHSVT =
RHS.getValueType();
623 RHS = GetScalarizedVector(
RHS);
627 return DAG.getNode(
N->getOpcode(),
DL,
LHS.getValueType(),
LHS,
RHS,
631SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(
SDNode *
N) {
636 if (
Op.getValueType() != EltVT)
644 N->getExtensionType(), SDLoc(
N),
N->getMemoryVT().getVectorElementType(),
645 N->getValueType(0).getVectorElementType(),
N->getChain(),
N->getBasePtr(),
656 N->getOpcode(), SDLoc(
N),
N->getMemoryVT().getVectorElementType(),
657 N->getChain(),
N->getBasePtr(), GetScalarizedVector(
N->getVal()),
665 assert(
N->isUnindexed() &&
"Indexed vector load?");
669 N->getValueType(0).getVectorElementType(), SDLoc(
N),
N->getChain(),
670 N->getBasePtr(), DAG.getPOISON(
N->getBasePtr().getValueType()),
671 N->getPointerInfo(),
N->getMemoryVT().getVectorElementType(),
672 N->getBaseAlign(),
N->getMemOperand()->getFlags(),
673 N->getMMOMetadataForSubAccess());
685 EVT OpVT =
Op.getValueType();
695 Op = GetScalarizedVector(
Op);
698 Op = DAG.getExtractVectorElt(
DL, VT,
Op, 0);
700 return DAG.getNode(
N->getOpcode(), SDLoc(
N), DestVT,
Op,
N->getFlags());
706 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
707 return DAG.getNode(
N->getOpcode(), SDLoc(
N), EltVT,
708 LHS, DAG.getValueType(ExtVT));
715 EVT OpVT =
Op.getValueType();
720 Op = GetScalarizedVector(
Op);
722 Op = DAG.getExtractVectorElt(
DL, OpEltVT,
Op, 0);
725 switch (
N->getOpcode()) {
737SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(
SDNode *
N) {
740 EVT OpVT =
Op.getValueType();
750 Op = GetScalarizedVector(
Op);
753 Op = DAG.getExtractVectorElt(
DL, VT,
Op, 0);
756 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
757 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
758 return DAG.getAddrSpaceCast(
DL, DestVT,
Op, SrcAS, DestAS,
759 AddrSpaceCastN->getFlags());
762SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(
SDNode *
N) {
773DAGTypeLegalizer::ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
774 assert(
N->getNumValues() ==
N->getNumOperands() &&
775 "Expected one result per operand");
779 for (
unsigned I = 0;
I !=
N->getNumValues(); ++
I)
780 SetScalarizedVector(
SDValue(
N,
I), GetScalarizedVector(
N->getOperand(
I)));
786 EVT OpVT =
Cond.getValueType();
795 Cond = DAG.getExtractVectorElt(
DL, VT,
Cond, 0);
798 SDValue LHS = GetScalarizedVector(
N->getOperand(1));
800 TLI.getBooleanContents(
false,
false);
807 if (TLI.getBooleanContents(
false,
false) !=
808 TLI.getBooleanContents(
false,
true)) {
812 EVT OpVT =
Cond->getOperand(0).getValueType();
814 VecBool = TLI.getBooleanContents(OpVT);
819 EVT CondVT =
Cond.getValueType();
820 if (ScalarBool != VecBool) {
821 switch (ScalarBool) {
829 Cond, DAG.getConstant(1, SDLoc(
N), CondVT));
836 Cond, DAG.getValueType(MVT::i1));
842 auto BoolVT = getSetCCResultType(CondVT);
843 if (BoolVT.bitsLT(CondVT))
846 return DAG.getSelect(SDLoc(
N),
LHS.getValueType(),
Cond,
LHS,
847 GetScalarizedVector(
N->getOperand(2)),
N->getFlags());
851 SDValue LHS = GetScalarizedVector(
N->getOperand(1));
852 return DAG.getSelect(SDLoc(
N),
853 LHS.getValueType(),
N->getOperand(0),
LHS,
854 GetScalarizedVector(
N->getOperand(2)));
858 SDValue LHS = GetScalarizedVector(
N->getOperand(2));
860 N->getOperand(0),
N->getOperand(1),
861 LHS, GetScalarizedVector(
N->getOperand(3)),
866 return DAG.getUNDEF(
N->getValueType(0).getVectorElementType());
869SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(
SDNode *
N) {
873 return DAG.getUNDEF(
N->getValueType(0).getVectorElementType());
875 return GetScalarizedVector(
N->getOperand(
Op));
878SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(
SDNode *
N) {
880 EVT SrcVT = Src.getValueType();
885 Src = GetScalarizedVector(Src);
889 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
891 EVT DstVT =
N->getValueType(0).getVectorElementType();
892 return DAG.getNode(
N->getOpcode(), dl, DstVT, Src,
N->getOperand(1));
896 assert(
N->getValueType(0).isVector() &&
897 N->getOperand(0).getValueType().isVector() &&
898 "Operand types must be vectors");
901 EVT OpVT =
LHS.getValueType();
902 EVT NVT =
N->getValueType(0).getVectorElementType();
907 LHS = GetScalarizedVector(
LHS);
908 RHS = GetScalarizedVector(
RHS);
911 LHS = DAG.getExtractVectorElt(
DL, VT,
LHS, 0);
912 RHS = DAG.getExtractVectorElt(
DL, VT,
RHS, 0);
922 return DAG.getNode(ExtendCode,
DL, NVT, Res);
933 Arg = GetScalarizedVector(Arg);
936 Arg = DAG.getExtractVectorElt(
DL, VT, Arg, 0);
945 return DAG.getNode(ExtendCode,
DL, ResultVT, Res);
952bool DAGTypeLegalizer::ScalarizeVectorOperand(
SDNode *
N,
unsigned OpNo) {
958 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
961 switch (
N->getOpcode()) {
964 dbgs() <<
"ScalarizeVectorOperand Op #" << OpNo <<
": ";
971 Res = ScalarizeVecOp_BITCAST(
N);
974 Res = ScalarizeVecOp_FAKE_USE(
N);
992 Res = ScalarizeVecOp_UnaryOp(
N);
997 Res = ScalarizeVecOp_UnaryOpWithExtraInput(
N);
1000 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
1001 "Unexpected vector type!");
1002 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
1004 N->getOpcode(), SDLoc(
N),
N->getValueType(0).getScalarType(), Elt,
1005 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
1013 Res = ScalarizeVecOp_UnaryOp_StrictFP(
N);
1016 Res = ScalarizeVecOp_CONCAT_VECTORS(
N);
1019 Res = ScalarizeVecOp_INSERT_SUBVECTOR(
N, OpNo);
1022 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(
N);
1025 Res = ScalarizeVecOp_VSELECT(
N);
1028 Res = ScalarizeVecOp_VSETCC(
N);
1032 Res = ScalarizeVecOp_VSTRICT_FSETCC(
N, OpNo);
1047 Res = ScalarizeVecOp_STRICT_FP_ROUND(
N, OpNo);
1050 Res = ScalarizeVecOp_FP_ROUND(
N, OpNo);
1053 Res = ScalarizeVecOp_STRICT_FP_EXTEND(
N);
1056 Res = ScalarizeVecOp_FP_EXTEND(
N);
1075 Res = ScalarizeVecOp_VECREDUCE(
N);
1079 Res = ScalarizeVecOp_VECREDUCE_SEQ(
N);
1083 Res = ScalarizeVecOp_CMP(
N);
1086 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(
N);
1090 Res = ScalarizeVecOp_CTTZ_ELTS(
N);
1093 Res = ScalarizeVecOp_VECTOR_MATCH(
N, OpNo);
1099 Res = ScalarizeVecOp_MaskedBinOp(
N, OpNo);
1104 if (!Res.
getNode())
return false;
1112 "Invalid operand expansion");
1114 ReplaceValueWith(
SDValue(
N, 0), Res);
1121 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
1123 N->getValueType(0), Elt);
1128 assert(
N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1129 "Fake Use: Unexpected vector type!");
1130 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
1131 return DAG.getNode(
ISD::FAKE_USE, SDLoc(), MVT::Other,
N->getOperand(0), Elt);
1137 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
1138 "Unexpected vector type!");
1139 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
1140 SDValue Op = DAG.getNode(
N->getOpcode(), SDLoc(
N),
1141 N->getValueType(0).getScalarType(), Elt);
1149SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(
SDNode *
N) {
1150 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
1151 "Unexpected vector type!");
1152 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
1154 DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getValueType(0).getScalarType(),
1155 Elt,
N->getOperand(1));
1163SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(
SDNode *
N) {
1164 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
1165 "Unexpected vector type!");
1166 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
1168 {
N->getValueType(0).getScalarType(), MVT::Other },
1169 {
N->getOperand(0), Elt });
1179 ReplaceValueWith(
SDValue(
N, 0), Res);
1184SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(
SDNode *
N) {
1186 for (
unsigned i = 0, e =
N->getNumOperands(); i < e; ++i)
1187 Ops[i] = GetScalarizedVector(
N->getOperand(i));
1188 return DAG.getBuildVector(
N->getValueType(0), SDLoc(
N),
Ops);
1193SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(
SDNode *
N,
1197 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
1198 SDValue ContainingVec =
N->getOperand(0);
1206SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
1207 EVT VT =
N->getValueType(0);
1208 SDValue Res = GetScalarizedVector(
N->getOperand(0));
1220 SDValue ScalarCond = GetScalarizedVector(
N->getOperand(0));
1221 EVT VT =
N->getValueType(0);
1223 return DAG.getNode(
ISD::SELECT, SDLoc(
N), VT, ScalarCond,
N->getOperand(1),
1232 assert(
N->getValueType(0).isVector() &&
1233 N->getOperand(0).getValueType().isVector() &&
1234 "Operand types must be vectors");
1235 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
1236 "Expected single-element vector type");
1238 EVT VT =
N->getValueType(0);
1239 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
1240 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
1242 EVT OpVT =
N->getOperand(0).getValueType();
1254 Res = DAG.
getNode(ExtendCode,
DL, NVT, Res);
1260SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(
SDNode *
N,
1262 assert(OpNo == 1 &&
"Wrong operand for scalarization!");
1263 assert(
N->getValueType(0).isVector() &&
1264 N->getOperand(1).getValueType().isVector() &&
1265 "Operand types must be vectors");
1266 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
1267 "Expected single-element vector type");
1269 EVT VT =
N->getValueType(0);
1271 SDValue LHS = GetScalarizedVector(
N->getOperand(1));
1272 SDValue RHS = GetScalarizedVector(
N->getOperand(2));
1275 EVT OpVT =
N->getOperand(1).getValueType();
1279 {Ch, LHS, RHS, CC});
1288 Res = DAG.
getNode(ExtendCode,
DL, NVT, Res);
1293 ReplaceValueWith(
SDValue(
N, 0), Res);
1300 assert(
N->isUnindexed() &&
"Indexed store of one-element vector?");
1301 assert(OpNo == 1 &&
"Do not know how to scalarize this operand!");
1304 if (
N->isTruncatingStore())
1305 return DAG.getTruncStore(
1306 N->getChain(), dl, GetScalarizedVector(
N->getOperand(1)),
1307 N->getBasePtr(),
N->getPointerInfo(),
1308 N->getMemoryVT().getVectorElementType(),
N->getBaseAlign(),
1309 N->getMemOperand()->getFlags(),
N->getMMOMetadataForSubAccess());
1311 return DAG.getStore(
N->getChain(), dl, GetScalarizedVector(
N->getOperand(1)),
1312 N->getBasePtr(),
N->getPointerInfo(),
N->getBaseAlign(),
1313 N->getMemOperand()->getFlags(),
1314 N->getMMOMetadataForSubAccess());
1320 SDValue ScalarVal = GetScalarizedVector(
N->getVal());
1322 N->getMemoryVT().getVectorElementType(),
N->getChain(),
1323 ScalarVal,
N->getBasePtr(),
N->getMemOperand());
1330 assert(OpNo == 3 &&
"Can only scalarize the mask operand!");
1331 EVT DataVT =
N->getValueType(0);
1332 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
1334 NewOps[OpNo] =
Mask;
1335 SDNode *Res = DAG.UpdateNodeOperands(
N, NewOps);
1349 assert(!
N->isCompressingStore() &&
"CompressingStore not expected");
1352 EVT DataVT =
N->getValue().getValueType();
1354 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
1355 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
1360SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(
SDNode *
N,
unsigned OpNo) {
1361 assert(OpNo == 0 &&
"Wrong operand for scalarization!");
1362 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
1364 N->getValueType(0).getVectorElementType(), Elt,
1369SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(
SDNode *
N,
1371 assert(OpNo == 1 &&
"Wrong operand for scalarization!");
1372 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
1375 {
N->getValueType(0).getVectorElementType(), MVT::Other},
1385 ReplaceValueWith(
SDValue(
N, 0), Res);
1392 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
1394 N->getValueType(0).getVectorElementType(), Elt);
1400SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(
SDNode *
N) {
1401 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
1404 {
N->getValueType(0).getVectorElementType(), MVT::Other},
1405 {
N->getOperand(0), Elt});
1414 ReplaceValueWith(
SDValue(
N, 0), Res);
1419 SDValue Res = GetScalarizedVector(
N->getOperand(0));
1426SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(
SDNode *
N) {
1432 SDValue Op = GetScalarizedVector(VecOp);
1433 return DAG.getNode(BaseOpc, SDLoc(
N),
N->getValueType(0),
1434 AccOp,
Op,
N->getFlags());
1438 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
1439 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
1446SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(
SDNode *
N) {
1454 EVT VT =
N->getValueType(0);
1455 return DAG.getConstant(0, SDLoc(
N), VT);
1462 return DAG.getConstant(0, SDLoc(
N),
N->getValueType(0));
1463 SDValue Op = GetScalarizedVector(
N->getOperand(0));
1465 DAG.getSetCC(SDLoc(
N), MVT::i1,
Op,
1466 DAG.getConstant(0, SDLoc(
N),
Op.getValueType()),
ISD::SETEQ);
1467 return DAG.getZExtOrTrunc(SetCC, SDLoc(
N),
N->getValueType(0));
1470SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(
SDNode *
N) {
1475 N->getValueType(0).getScalarType(), Mask,
1476 DAG.getVectorIdxConstant(0,
DL));
1481 return TLI.expandVectorMatch(
N, DAG);
1484SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(
SDNode *
N,
unsigned OpNo) {
1485 assert(OpNo == 2 &&
"Can only scalarize mask operand");
1488 SDValue LHS = DAG.getExtractVectorElt(
DL, VT,
N->getOperand(0), 0);
1489 SDValue RHS = DAG.getExtractVectorElt(
DL, VT,
N->getOperand(1), 0);
1498 DAG.getSelect(
DL, VT, Mask,
RHS, DAG.getConstant(1,
DL, VT)));
1510void DAGTypeLegalizer::SplitVectorResult(
SDNode *
N,
unsigned ResNo) {
1515 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
1518 switch (
N->getOpcode()) {
1521 dbgs() <<
"SplitVectorResult #" << ResNo <<
": ";
1530 SplitVecRes_LOOP_DEPENDENCE_MASK(
N,
Lo,
Hi);
1533 SplitVecRes_MASK_BEFOREFIRST(
N,
Lo,
Hi);
1540 case ISD::VP_MERGE: SplitRes_Select(
N,
Lo,
Hi);
break;
1556 SplitVecRes_ScalarOp(
N,
Lo,
Hi);
1559 SplitVecRes_STEP_VECTOR(
N,
Lo,
Hi);
1571 case ISD::VP_LOAD_FF:
1574 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1581 case ISD::VP_GATHER:
1585 SplitVecRes_VECTOR_COMPRESS(
N,
Lo,
Hi);
1588 SplitVecRes_VECTOR_SHUFFLE_VAR(
N,
Lo,
Hi);
1591 SplitVecRes_SETCC(
N,
Lo,
Hi);
1594 SplitVecRes_VECTOR_REPEAT(
N,
Lo,
Hi);
1597 SplitVecRes_VECTOR_REVERSE(
N,
Lo,
Hi);
1604 SplitVecRes_VECTOR_SPLICE(
N,
Lo,
Hi);
1607 SplitVecRes_VECTOR_DEINTERLEAVE(
N);
1610 SplitVecRes_VECTOR_INTERLEAVE(
N);
1613 SplitVecRes_VAARG(
N,
Lo,
Hi);
1619 SplitVecRes_ExtVecInRegOp(
N,
Lo,
Hi);
1673 SplitVecRes_UnaryOp(
N,
Lo,
Hi);
1676 SplitVecRes_ADDRSPACECAST(
N,
Lo,
Hi);
1682 SplitVecRes_UnaryOpWithTwoResults(
N, ResNo,
Lo,
Hi);
1688 SplitVecRes_ExtendOp(
N,
Lo,
Hi);
1744 SplitVecRes_BinOp(
N,
Lo,
Hi);
1750 SplitVecRes_MaskedBinOp(
N,
Lo,
Hi);
1755 SplitVecRes_TernaryOp(
N,
Lo,
Hi);
1759 SplitVecRes_CMP(
N,
Lo,
Hi);
1762#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1763 case ISD::STRICT_##DAGN:
1764#include "llvm/IR/ConstrainedOps.def"
1765 SplitVecRes_StrictFPOp(
N,
Lo,
Hi);
1770 SplitVecRes_FP_TO_XINT_SAT(
N,
Lo,
Hi);
1779 SplitVecRes_OverflowOp(
N, ResNo,
Lo,
Hi);
1789 SplitVecRes_FIX(
N,
Lo,
Hi);
1791 case ISD::EXPERIMENTAL_VP_SPLICE:
1792 SplitVecRes_VP_SPLICE(
N,
Lo,
Hi);
1794 case ISD::EXPERIMENTAL_VP_REVERSE:
1795 SplitVecRes_VP_REVERSE(
N,
Lo,
Hi);
1801 SplitVecRes_PARTIAL_REDUCE_MLA(
N,
Lo,
Hi);
1804 SplitVecRes_GET_ACTIVE_LANE_MASK(
N,
Lo,
Hi);
1807 SplitVecRes_VECTOR_MATCH(
N,
Lo,
Hi);
1816void DAGTypeLegalizer::IncrementPointer(
MemSDNode *
N,
EVT MemVT,
1823 SDValue BytesIncrement = DAG.getVScale(
1826 MPI = MachinePointerInfo(
N->getPointerInfo().getAddrSpace());
1828 *ScaledOffset += IncrementSize;
1838std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(
SDValue Mask) {
1839 return SplitMask(Mask, SDLoc(Mask));
1842std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(
SDValue Mask,
1845 EVT MaskVT =
Mask.getValueType();
1847 GetSplitVector(Mask, MaskLo, MaskHi);
1849 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask,
DL);
1850 return std::make_pair(MaskLo, MaskHi);
1855 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1857 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
1860 const SDNodeFlags
Flags =
N->getFlags();
1861 unsigned Opcode =
N->getOpcode();
1862 if (
N->getNumOperands() == 2) {
1863 Lo = DAG.getNode(Opcode, dl, LHSLo.
getValueType(), LHSLo, RHSLo, Flags);
1864 Hi = DAG.getNode(Opcode, dl, LHSHi.
getValueType(), LHSHi, RHSHi, Flags);
1868 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1869 assert((
N->getOpcode() == ISD::VP_UDIV ||
N->getOpcode() == ISD::VP_SDIV ||
1870 N->getOpcode() == ISD::VP_UREM ||
N->getOpcode() == ISD::VP_SREM) &&
1871 "Expected VP opcode");
1874 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(2));
1877 std::tie(EVLLo, EVLHi) =
1878 DAG.SplitEVL(
N->getOperand(3),
N->getValueType(0), dl);
1881 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1883 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1889 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1891 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
1895 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
1897 std::tie(MaskLo, MaskHi) = SplitMask(Mask);
1901 const SDNodeFlags
Flags =
N->getFlags();
1902 unsigned Opcode =
N->getOpcode();
1903 Lo = DAG.getNode(Opcode, dl, LHSLo.
getValueType(), LHSLo, RHSLo, MaskLo,
1905 Hi = DAG.getNode(Opcode, dl, LHSHi.
getValueType(), LHSHi, RHSHi, MaskHi,
1912 GetSplitVector(
N->getOperand(0), Op0Lo, Op0Hi);
1914 GetSplitVector(
N->getOperand(1), Op1Lo, Op1Hi);
1916 GetSplitVector(
N->getOperand(2), Op2Lo, Op2Hi);
1919 const SDNodeFlags
Flags =
N->getFlags();
1920 unsigned Opcode =
N->getOpcode();
1922 DAG.getNode(Opcode, dl, Op0Lo.
getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1924 DAG.getNode(Opcode, dl, Op0Hi.
getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1928 LLVMContext &Ctxt = *DAG.getContext();
1934 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1936 GetSplitVector(
LHS, LHSLo, LHSHi);
1937 GetSplitVector(
RHS, RHSLo, RHSHi);
1939 std::tie(LHSLo, LHSHi) = DAG.SplitVector(
LHS, dl);
1940 std::tie(RHSLo, RHSHi) = DAG.SplitVector(
RHS, dl);
1944 Lo = DAG.getNode(
N->getOpcode(), dl, SplitResVT, LHSLo, RHSLo);
1945 Hi = DAG.getNode(
N->getOpcode(), dl, SplitResVT, LHSHi, RHSHi);
1950 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1952 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
1956 unsigned Opcode =
N->getOpcode();
1957 Lo = DAG.getNode(Opcode, dl, LHSLo.
getValueType(), LHSLo, RHSLo, Op2,
1959 Hi = DAG.getNode(Opcode, dl, LHSHi.
getValueType(), LHSHi, RHSHi, Op2,
1968 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
1975 switch (getTypeAction(InVT)) {
1989 GetExpandedOp(InOp,
Lo,
Hi);
1990 if (DAG.getDataLayout().isBigEndian())
2000 GetSplitVector(InOp,
Lo,
Hi);
2009 auto [InLo, InHi] = DAG.SplitVectorOperand(
N, 0);
2018 if (DAG.getDataLayout().isBigEndian())
2021 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT,
Lo,
Hi);
2023 if (DAG.getDataLayout().isBigEndian())
2029void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(
SDNode *
N,
SDValue &
Lo,
2035 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2038 Lo = DAG.getNode(
N->getOpcode(),
DL, LoVT, PtrA, PtrB,
2043 unsigned LaneOffset =
2046 Hi = DAG.getNode(
N->getOpcode(),
DL, HiVT, PtrA, PtrB,
2048 DAG.getConstant(LaneOffset,
DL, MVT::i64));
2055 GetSplitVector(
N->getOperand(0), InLo, InHi);
2062 getSetCCResultType(MVT::i1), MVT::i1);
2071 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2074 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
2077 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
2082 assert(!(
N->getNumOperands() & 1) &&
"Unsupported CONCAT_VECTORS");
2084 unsigned NumSubvectors =
N->getNumOperands() / 2;
2085 if (NumSubvectors == 1) {
2086 Lo =
N->getOperand(0);
2087 Hi =
N->getOperand(1);
2092 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2101void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(
SDNode *
N,
SDValue &
Lo,
2108 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2123 GetSplitVector(Vec,
Lo,
Hi);
2126 EVT LoVT =
Lo.getValueType();
2135 unsigned IdxVal =
Idx->getAsZExtVal();
2136 if (IdxVal + SubElems <= LoElems) {
2144 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
2146 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
2152 SDValue WideSubVec = GetWidenedVector(SubVec);
2154 std::tie(
Lo,
Hi) = DAG.SplitVector(WideSubVec, SDLoc(WideSubVec));
2162 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
2164 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
2165 auto &MF = DAG.getMachineFunction();
2169 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2174 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
2175 Store = DAG.getStore(
Store, dl, SubVec, SubVecPtr,
2179 Lo = DAG.getLoad(
Lo.getValueType(), dl,
Store, StackPtr, PtrInfo,
2184 MachinePointerInfo MPI =
Load->getPointerInfo();
2185 IncrementPointer(
Load, LoVT, MPI, StackPtr);
2188 Hi = DAG.getLoad(
Hi.getValueType(), dl,
Store, StackPtr, MPI, SmallestAlign);
2197 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
2202 EVT RHSVT =
RHS.getValueType();
2205 GetSplitVector(
RHS, RHSLo, RHSHi);
2207 std::tie(RHSLo, RHSHi) = DAG.SplitVector(
RHS, SDLoc(
RHS));
2222 SDValue FpValue =
N->getOperand(0);
2224 GetSplitVector(FpValue, ArgLo, ArgHi);
2226 std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue));
2228 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2237 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
2241 std::tie(LoVT, HiVT) =
2245 DAG.getValueType(LoVT));
2247 DAG.getValueType(HiVT));
2252 unsigned Opcode =
N->getOpcode();
2259 GetSplitVector(N0, InLo, InHi);
2261 std::tie(InLo, InHi) = DAG.SplitVectorOperand(
N, 0);
2266 EVT OutLoVT, OutHiVT;
2267 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2269 assert((2 * OutNumElements) <= InNumElements &&
2270 "Illegal extend vector in reg split");
2279 SmallVector<int, 8> SplitHi(InNumElements, -1);
2280 for (
unsigned i = 0; i != OutNumElements; ++i)
2281 SplitHi[i] = i + OutNumElements;
2282 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getPOISON(InLoVT), SplitHi);
2284 Lo = DAG.
getNode(Opcode, dl, OutLoVT, InLo);
2285 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
2290 unsigned NumOps =
N->getNumOperands();
2294 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2304 for (
unsigned i = 1; i <
NumOps; ++i) {
2309 EVT InVT =
Op.getValueType();
2314 GetSplitVector(
Op, OpLo, OpHi);
2316 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(
N, i);
2323 EVT LoValueVTs[] = {LoVT, MVT::Other};
2324 EVT HiValueVTs[] = {HiVT, MVT::Other};
2325 Lo = DAG.
getNode(
N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
2327 Hi = DAG.getNode(
N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
2333 Lo.getValue(1),
Hi.getValue(1));
2337 ReplaceValueWith(
SDValue(
N, 1), Chain);
2340SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(
SDNode *
N,
unsigned ResNE) {
2341 EVT VT =
N->getValueType(0);
2356 else if (NE > ResNE)
2360 SDVTList ChainVTs = DAG.getVTList(EltVT, MVT::Other);
2364 for (i = 0; i !=
NE; ++i) {
2366 for (
unsigned j = 1, e =
N->getNumOperands(); j != e; ++j) {
2367 SDValue Operand =
N->getOperand(j);
2371 Operands[
j] = DAG.getExtractVectorElt(dl, OperandEltVT, Operand, i);
2377 DAG.getNode(
N->getOpcode(), dl, ChainVTs,
Operands,
N->getFlags());
2385 for (; i < ResNE; ++i)
2386 Scalars.
push_back(DAG.getPOISON(EltVT));
2390 ReplaceValueWith(
SDValue(
N, 1), Chain);
2394 return DAG.getBuildVector(VecVT, dl, Scalars);
2397void DAGTypeLegalizer::SplitVecRes_OverflowOp(
SDNode *
N,
unsigned ResNo,
2400 EVT ResVT =
N->getValueType(0);
2401 EVT OvVT =
N->getValueType(1);
2402 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2403 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
2404 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
2406 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2408 GetSplitVector(
N->getOperand(0), LoLHS, HiLHS);
2409 GetSplitVector(
N->getOperand(1), LoRHS, HiRHS);
2411 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(
N, 0);
2412 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(
N, 1);
2415 unsigned Opcode =
N->getOpcode();
2416 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
2417 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
2419 DAG.getNode(Opcode, dl, LoVTs, {LoLHS, LoRHS},
N->getFlags()).getNode();
2421 DAG.getNode(Opcode, dl, HiVTs, {HiLHS, HiRHS},
N->getFlags()).getNode();
2427 unsigned OtherNo = 1 - ResNo;
2428 EVT OtherVT =
N->getValueType(OtherNo);
2430 SetSplitVector(
SDValue(
N, OtherNo),
2436 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
2440void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(
SDNode *
N,
SDValue &
Lo,
2446 GetSplitVector(Vec,
Lo,
Hi);
2449 unsigned IdxVal = CIdx->getZExtValue();
2450 unsigned LoNumElts =
Lo.getValueType().getVectorMinNumElements();
2451 if (IdxVal < LoNumElts) {
2453 Lo.getValueType(),
Lo, Elt, Idx);
2456 Hi = DAG.getInsertVectorElt(dl,
Hi, Elt, IdxVal - LoNumElts);
2476 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
2478 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
2479 auto &MF = DAG.getMachineFunction();
2483 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2488 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2489 Store = DAG.getTruncStore(
2495 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
2498 Lo = DAG.getLoad(LoVT, dl,
Store, StackPtr, PtrInfo, SmallestAlign);
2502 MachinePointerInfo MPI =
Load->getPointerInfo();
2503 IncrementPointer(
Load, LoVT, MPI, StackPtr);
2505 Hi = DAG.getLoad(HiVT, dl,
Store, StackPtr, MPI, SmallestAlign);
2508 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2509 if (LoVT !=
Lo.getValueType())
2511 if (HiVT !=
Hi.getValueType())
2519 assert(
N->getValueType(0).isScalableVector() &&
2520 "Only scalable vectors are supported for STEP_VECTOR");
2521 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2542 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2543 Lo = DAG.getNode(
N->getOpcode(), dl, LoVT,
N->getOperand(0));
2545 Hi = DAG.getPOISON(HiVT);
2555 "Extended load during type legalization!");
2557 EVT VT =
LD->getValueType(0);
2559 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2567 SDValue ALD = DAG.getAtomicLoad(
LD->getExtensionType(), dl, MemIntVT, IntVT,
2568 Ch, Ptr,
LD->getMemOperand());
2573 SplitInteger(ALD, LoIntVT, HiIntVT, ExtractLo, ExtractHi);
2575 Lo = DAG.getBitcast(LoVT, ExtractLo);
2576 Hi = DAG.getBitcast(HiVT, ExtractHi);
2588 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
LD->getValueType(0));
2594 EVT MemoryVT =
LD->getMemoryVT();
2596 MMOMetadata
Metadata =
LD->getMMOMetadataForSubAccess();
2598 EVT LoMemVT, HiMemVT;
2599 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2603 std::tie(
Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2604 std::tie(
Lo,
Hi) = DAG.SplitVector(
Value, dl);
2605 ReplaceValueWith(
SDValue(LD, 1), NewChain);
2610 LD->getPointerInfo(), LoMemVT,
LD->getBaseAlign(), MMOFlags,
2613 MachinePointerInfo MPI;
2614 IncrementPointer(LD, LoMemVT, MPI, Ptr);
2617 HiMemVT,
LD->getBaseAlign(), MMOFlags,
Metadata);
2626 ReplaceValueWith(
SDValue(LD, 1), Ch);
2631 assert(
LD->isUnindexed() &&
"Indexed VP load during type legalization!");
2634 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
LD->getValueType(0));
2640 assert(
Offset.isUndef() &&
"Unexpected indexed variable-length load offset");
2644 EVT MemoryVT =
LD->getMemoryVT();
2646 EVT LoMemVT, HiMemVT;
2647 bool HiIsEmpty =
false;
2648 std::tie(LoMemVT, HiMemVT) =
2649 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2654 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
2657 GetSplitVector(Mask, MaskLo, MaskHi);
2659 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2664 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL,
LD->getValueType(0), dl);
2666 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2669 MMOMetadata(
LD->getAAInfo(),
LD->getRanges()));
2672 DAG.getLoadVP(
LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr,
Offset,
2673 MaskLo, EVLLo, LoMemVT, MMO,
LD->isExpandingLoad());
2681 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2682 LD->isExpandingLoad());
2684 MachinePointerInfo MPI;
2686 MPI = MachinePointerInfo(
LD->getPointerInfo().getAddrSpace());
2688 MPI =
LD->getPointerInfo().getWithOffset(
2691 MMO = DAG.getMachineFunction().getMachineMemOperand(
2693 Alignment, MMOMetadata(
LD->getAAInfo(),
LD->getRanges()));
2695 Hi = DAG.getLoadVP(
LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
2696 Offset, MaskHi, EVLHi, HiMemVT, MMO,
2697 LD->isExpandingLoad());
2707 ReplaceValueWith(
SDValue(LD, 1), Ch);
2713 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(
LD->getValueType(0));
2724 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
2727 GetSplitVector(Mask, MaskLo, MaskHi);
2729 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2733 auto [EVLLo, EVLHi] = DAG.SplitEVL(EVL,
LD->getValueType(0), dl);
2735 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2738 MMOMetadata(
LD->getAAInfo(),
LD->getRanges()));
2740 Lo = DAG.getLoadFFVP(LoVT, dl, Ch, Ptr, MaskLo, EVLLo, MMO);
2743 Hi = DAG.getPOISON(HiVT);
2745 ReplaceValueWith(
SDValue(LD, 1),
Lo.getValue(1));
2746 ReplaceValueWith(
SDValue(LD, 2),
Lo.getValue(2));
2752 "Indexed VP strided load during type legalization!");
2754 "Unexpected indexed variable-length load offset");
2759 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->
getValueType(0));
2761 EVT LoMemVT, HiMemVT;
2762 bool HiIsEmpty =
false;
2763 std::tie(LoMemVT, HiMemVT) =
2764 DAG.GetDependentSplitDestVTs(SLD->
getMemoryVT(), LoVT, &HiIsEmpty);
2769 SplitVecRes_SETCC(
Mask.getNode(), LoMask, HiMask);
2772 GetSplitVector(Mask, LoMask, HiMask);
2774 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask,
DL);
2778 std::tie(LoEVL, HiEVL) =
2782 Lo = DAG.getStridedLoadVP(
2809 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2816 SLD->
getStride(), HiMask, HiEVL, HiMemVT, MMO,
2827 ReplaceValueWith(
SDValue(SLD, 1), Ch);
2835 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->
getValueType(0));
2840 assert(
Offset.isUndef() &&
"Unexpected indexed masked load offset");
2850 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
2853 GetSplitVector(Mask, MaskLo, MaskHi);
2855 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2859 EVT LoMemVT, HiMemVT;
2860 bool HiIsEmpty =
false;
2861 std::tie(LoMemVT, HiMemVT) =
2862 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2864 SDValue PassThruLo, PassThruHi;
2866 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2868 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2870 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2875 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr,
Offset, MaskLo, PassThruLo, LoMemVT,
2885 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2888 MachinePointerInfo MPI;
2895 MMO = DAG.getMachineFunction().getMachineMemOperand(
2900 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr,
Offset, MaskHi, PassThruHi,
2912 ReplaceValueWith(
SDValue(MLD, 1), Ch);
2920 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
2930 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2933 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2936 EVT MemoryVT =
N->getMemoryVT();
2942 SplitVecRes_SETCC(
Ops.Mask.getNode(), MaskLo, MaskHi);
2944 std::tie(MaskLo, MaskHi) = SplitMask(
Ops.Mask, dl);
2947 EVT LoMemVT, HiMemVT;
2949 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2952 if (getTypeAction(
Ops.Index.getValueType()) ==
2954 GetSplitVector(
Ops.Index, IndexLo, IndexHi);
2956 std::tie(IndexLo, IndexHi) = DAG.SplitVector(
Ops.Index, dl);
2959 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2961 Alignment, MMOMetadata(
N->getAAInfo(),
N->getRanges()));
2964 SDValue PassThru = MGT->getPassThru();
2965 SDValue PassThruLo, PassThruHi;
2968 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2970 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2975 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo,
Ops.Scale};
2976 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2977 OpsLo, MMO, IndexTy, ExtType);
2979 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi,
Ops.Scale};
2980 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2981 OpsHi, MMO, IndexTy, ExtType);
2985 std::tie(EVLLo, EVLHi) =
2986 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2988 SDValue OpsLo[] = {Ch, Ptr, IndexLo,
Ops.Scale, MaskLo, EVLLo};
2989 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2990 MMO, VPGT->getIndexType());
2992 SDValue OpsHi[] = {Ch, Ptr, IndexHi,
Ops.Scale, MaskHi, EVLHi};
2993 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2994 MMO, VPGT->getIndexType());
3004 ReplaceValueWith(
SDValue(
N, 1), Ch);
3018 EVT VecVT =
N->getValueType(0);
3020 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VecVT);
3021 bool HasCustomLowering =
false;
3028 HasCustomLowering =
true;
3034 SDValue Passthru =
N->getOperand(2);
3035 if (!HasCustomLowering) {
3036 SDValue Compressed = TLI.expandVECTOR_COMPRESS(
N, DAG);
3037 std::tie(
Lo,
Hi) = DAG.SplitVector(Compressed,
DL, LoVT, HiVT);
3044 std::tie(
Lo,
Hi) = DAG.SplitVectorOperand(
N, 0);
3045 std::tie(LoMask, HiMask) = SplitMask(Mask);
3047 SDValue UndefPassthru = DAG.getPOISON(LoVT);
3052 VecVT.
getStoreSize(), DAG.getReducedAlign(VecVT,
false));
3065 Offset = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Offset);
3067 SDValue Chain = DAG.getEntryNode();
3068 Chain = DAG.getStore(Chain,
DL,
Lo, StackPtr, PtrInfo);
3072 SDValue Compressed = DAG.getLoad(VecVT,
DL, Chain, StackPtr, PtrInfo);
3076 EVT MaskVT =
Mask.getValueType();
3083 SDValue StepVector = DAG.getStepVector(
DL, WideMaskVT);
3084 SDValue SplatNumActiveElts = DAG.getSplat(WideMaskVT,
DL, NumActiveElts);
3086 DAG.getSetCC(
DL, MaskVT, StepVector, SplatNumActiveElts,
ISD::SETULT);
3089 Compressed, Passthru);
3091 std::tie(
Lo,
Hi) = DAG.SplitVector(Compressed,
DL);
3094void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE_VAR(
SDNode *
N,
SDValue &
Lo,
3097 SDValue Res = TLI.expandVECTOR_SHUFFLE_VAR(
N, DAG);
3098 std::tie(
Lo,
Hi) = DAG.SplitVector(Res, SDLoc(
N));
3102 assert(
N->getValueType(0).isVector() &&
3103 N->getOperand(0).getValueType().isVector() &&
3104 "Operand types must be vectors");
3108 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
3112 if (getTypeAction(
N->getOperand(0).getValueType()) ==
3114 GetSplitVector(
N->getOperand(0), LL, LH);
3116 std::tie(LL, LH) = DAG.SplitVectorOperand(
N, 0);
3118 if (getTypeAction(
N->getOperand(1).getValueType()) ==
3120 GetSplitVector(
N->getOperand(1), RL, RH);
3122 std::tie(RL, RH) = DAG.SplitVectorOperand(
N, 1);
3124 Lo = DAG.getNode(
N->getOpcode(),
DL, LoVT, LL, RL,
N->getOperand(2));
3125 Hi = DAG.getNode(
N->getOpcode(),
DL, HiVT, LH, RH,
N->getOperand(2));
3133 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
3137 EVT InVT =
N->getOperand(0).getValueType();
3139 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3141 std::tie(
Lo,
Hi) = DAG.SplitVectorOperand(
N, 0);
3143 const SDNodeFlags
Flags =
N->getFlags();
3144 unsigned Opcode =
N->getOpcode();
3146 Lo = DAG.getNode(Opcode, dl, LoVT,
Lo,
N->getOperand(1),
N->getOperand(2),
3147 N->getOperand(3), Flags);
3148 Hi = DAG.getNode(Opcode, dl, HiVT,
Hi,
N->getOperand(1),
N->getOperand(2),
3149 N->getOperand(3), Flags);
3155 Lo = DAG.getNode(Opcode, dl, LoVT,
Lo,
N->getOperand(1), Flags);
3156 Hi = DAG.getNode(Opcode, dl, HiVT,
Hi,
N->getOperand(1), Flags);
3158 Lo = DAG.getNode(Opcode, dl, LoVT,
Lo, Flags);
3159 Hi = DAG.getNode(Opcode, dl, HiVT,
Hi, Flags);
3166 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(
N->getValueType(0));
3170 EVT InVT =
N->getOperand(0).getValueType();
3172 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3174 std::tie(
Lo,
Hi) = DAG.SplitVectorOperand(
N, 0);
3177 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3178 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3179 SDNodeFlags
Flags = AddrSpaceCastN->getFlags();
3180 Lo = DAG.getAddrSpaceCast(dl, LoVT,
Lo, SrcAS, DestAS, Flags);
3181 Hi = DAG.getAddrSpaceCast(dl, HiVT,
Hi, SrcAS, DestAS, Flags);
3184void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(
SDNode *
N,
3189 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(
N->getValueType(0));
3190 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(
N->getValueType(1));
3194 EVT InVT =
N->getOperand(0).getValueType();
3196 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3198 std::tie(
Lo,
Hi) = DAG.SplitVectorOperand(
N, 0);
3200 Lo = DAG.getNode(
N->getOpcode(), dl, {LoVT, LoVT1},
Lo,
N->getFlags());
3201 Hi = DAG.getNode(
N->getOpcode(), dl, {HiVT, HiVT1},
Hi,
N->getFlags());
3203 SDNode *HiNode =
Hi.getNode();
3204 SDNode *LoNode =
Lo.getNode();
3207 unsigned OtherNo = 1 - ResNo;
3208 EVT OtherVT =
N->getValueType(OtherNo);
3216 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
3223 EVT SrcVT =
N->getOperand(0).getValueType();
3224 EVT DestVT =
N->getValueType(0);
3226 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
3243 LLVMContext &Ctx = *DAG.getContext();
3247 EVT SplitLoVT, SplitHiVT;
3248 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
3249 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
3250 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
3251 LLVM_DEBUG(
dbgs() <<
"Split vector extend via incremental extend:";
3252 N->dump(&DAG);
dbgs() <<
"\n");
3255 DAG.getNode(
N->getOpcode(), dl, NewSrcVT,
N->getOperand(0));
3257 std::tie(
Lo,
Hi) = DAG.SplitVector(NewSrc, dl);
3259 Lo = DAG.getNode(
N->getOpcode(), dl, LoVT,
Lo);
3260 Hi = DAG.getNode(
N->getOpcode(), dl, HiVT,
Hi);
3265 SplitVecRes_UnaryOp(
N,
Lo,
Hi);
3273 GetSplitVector(
N->getOperand(0), Inputs[0], Inputs[1]);
3274 GetSplitVector(
N->getOperand(1), Inputs[2], Inputs[3]);
3283 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &
DL](
SDValue &Input1,
3285 ArrayRef<int>
Mask) {
3288 "Expected build vector node.");
3291 for (
unsigned I = 0;
I < NewElts; ++
I) {
3296 Ops[
I] = Input2.getOperand(Idx - NewElts);
3298 Ops[
I] = Input1.getOperand(Idx);
3303 return DAG.getBuildVector(NewVT,
DL,
Ops);
3309 SmallVector<int> OrigMask(
N->getMask());
3311 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT,
this, NewElts,
3312 &
DL](SmallVectorImpl<int> &
Mask) {
3314 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3315 for (
unsigned Idx = 0;
Idx < std::size(Inputs); ++
Idx) {
3319 Input.getOperand(0).getValueType() !=
Input.getValueType())
3321 ShufflesIdxs[std::make_pair(
Input.getOperand(0),
Input.getOperand(1))]
3323 ShufflesIdxs[std::make_pair(
Input.getOperand(1),
Input.getOperand(0))]
3326 for (
auto &
P : ShufflesIdxs) {
3327 if (
P.second.size() < 2)
3331 for (
int &Idx : Mask) {
3334 unsigned SrcRegIdx =
Idx / NewElts;
3335 if (Inputs[SrcRegIdx].
isUndef()) {
3343 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3348 Idx = MaskElt % NewElts +
3349 P.second[Shuffle->getOperand(MaskElt / NewElts) ==
P.first.first
3355 Inputs[
P.second[0]] =
P.first.first;
3356 Inputs[
P.second[1]] =
P.first.second;
3359 ShufflesIdxs[std::make_pair(
P.first.second,
P.first.first)].clear();
3362 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3363 for (
int &Idx : Mask) {
3366 unsigned SrcRegIdx =
Idx / NewElts;
3367 if (Inputs[SrcRegIdx].
isUndef()) {
3374 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3375 !Inputs[SrcRegIdx].getOperand(1).
isUndef() &&
3378 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3380 if (UsedSubVector.count() > 1) {
3382 for (
unsigned I = 0;
I < std::size(Inputs); ++
I) {
3383 if (UsedSubVector.test(2 *
I) == UsedSubVector.test(2 *
I + 1))
3385 if (Pairs.
empty() || Pairs.
back().size() == 2)
3387 if (UsedSubVector.test(2 *
I)) {
3388 Pairs.
back().emplace_back(
I, 0);
3390 assert(UsedSubVector.test(2 *
I + 1) &&
3391 "Expected to be used one of the subvectors.");
3392 Pairs.
back().emplace_back(
I, 1);
3395 if (!Pairs.
empty() && Pairs.
front().size() > 1) {
3397 for (
int &Idx : Mask) {
3400 unsigned SrcRegIdx =
Idx / NewElts;
3402 Pairs, [SrcRegIdx](
ArrayRef<std::pair<unsigned, int>> Idxs) {
3403 return Idxs.front().first == SrcRegIdx ||
3404 Idxs.back().first == SrcRegIdx;
3406 if (It == Pairs.
end())
3408 Idx = It->front().first * NewElts + (
Idx % NewElts) % (NewElts / 2) +
3409 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3412 for (
ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3413 Inputs[Idxs.front().first] = DAG.
getNode(
3415 Inputs[Idxs.front().first].getValueType(),
3416 Inputs[Idxs.front().first].getOperand(Idxs.front().second),
3417 Inputs[Idxs.back().first].getOperand(Idxs.back().second));
3426 for (
unsigned I = 0;
I < std::size(Inputs); ++
I) {
3430 if (Shuffle->getOperand(0).getValueType() != NewVT)
3433 if (!Inputs[
I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
3434 !Shuffle->isSplat()) {
3436 }
else if (!Inputs[
I].hasOneUse() &&
3437 !Shuffle->getOperand(1).isUndef()) {
3439 for (
int &Idx : Mask) {
3442 unsigned SrcRegIdx =
Idx / NewElts;
3445 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3450 int OpIdx = MaskElt / NewElts;
3463 for (
int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3464 if (Shuffle->getOperand(OpIdx).isUndef())
3466 auto *It =
find(Inputs, Shuffle->getOperand(OpIdx));
3467 if (It == std::end(Inputs))
3469 int FoundOp = std::distance(std::begin(Inputs), It);
3472 for (
int &Idx : Mask) {
3475 unsigned SrcRegIdx =
Idx / NewElts;
3478 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3483 int MaskIdx = MaskElt / NewElts;
3484 if (OpIdx == MaskIdx)
3485 Idx = MaskElt % NewElts + FoundOp * NewElts;
3488 Op = (OpIdx + 1) % 2;
3496 for (
int &Idx : Mask) {
3499 unsigned SrcRegIdx =
Idx / NewElts;
3502 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
3503 int OpIdx = MaskElt / NewElts;
3506 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3512 TryPeekThroughShufflesInputs(OrigMask);
3514 auto &&MakeUniqueInputs = [&Inputs, &
IsConstant,
3515 NewElts](SmallVectorImpl<int> &
Mask) {
3516 SetVector<SDValue> UniqueInputs;
3517 SetVector<SDValue> UniqueConstantInputs;
3518 for (
const auto &
I : Inputs) {
3520 UniqueConstantInputs.
insert(
I);
3521 else if (!
I.isUndef())
3526 if (UniqueInputs.
size() != std::size(Inputs)) {
3527 auto &&UniqueVec = UniqueInputs.
takeVector();
3528 auto &&UniqueConstantVec = UniqueConstantInputs.
takeVector();
3529 unsigned ConstNum = UniqueConstantVec.size();
3530 for (
int &Idx : Mask) {
3533 unsigned SrcRegIdx =
Idx / NewElts;
3534 if (Inputs[SrcRegIdx].
isUndef()) {
3538 const auto It =
find(UniqueConstantVec, Inputs[SrcRegIdx]);
3539 if (It != UniqueConstantVec.end()) {
3541 NewElts * std::distance(UniqueConstantVec.begin(), It);
3542 assert(Idx >= 0 &&
"Expected defined mask idx.");
3545 const auto RegIt =
find(UniqueVec, Inputs[SrcRegIdx]);
3546 assert(RegIt != UniqueVec.end() &&
"Cannot find non-const value.");
3548 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
3549 assert(Idx >= 0 &&
"Expected defined mask idx.");
3551 copy(UniqueConstantVec, std::begin(Inputs));
3552 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
3555 MakeUniqueInputs(OrigMask);
3557 copy(Inputs, std::begin(OrigInputs));
3563 unsigned FirstMaskIdx =
High * NewElts;
3566 assert(!Output &&
"Expected default initialized initial value.");
3567 TryPeekThroughShufflesInputs(Mask);
3568 MakeUniqueInputs(Mask);
3570 copy(Inputs, std::begin(TmpInputs));
3573 bool SecondIteration =
false;
3574 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](
unsigned Idx) {
3579 if (UsedIdx >= 0 &&
static_cast<unsigned>(UsedIdx) == Idx)
3580 SecondIteration =
true;
3581 return SecondIteration;
3584 Mask, std::size(Inputs), std::size(Inputs),
3586 [&Output, &DAG = DAG, NewVT]() {
Output = DAG.getPOISON(NewVT); },
3587 [&
Output, &DAG = DAG, NewVT, &
DL, &Inputs,
3588 &BuildVector](ArrayRef<int>
Mask,
unsigned Idx,
unsigned ) {
3590 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3592 Output = DAG.getVectorShuffle(NewVT,
DL, Inputs[Idx],
3593 DAG.getPOISON(NewVT), Mask);
3596 [&AccumulateResults, &
Output, &DAG = DAG, NewVT, &
DL, &Inputs,
3597 &TmpInputs, &BuildVector](ArrayRef<int>
Mask,
unsigned Idx1,
3598 unsigned Idx2,
bool ) {
3599 if (AccumulateResults(Idx1)) {
3602 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3604 Output = DAG.getVectorShuffle(NewVT,
DL, Inputs[Idx1],
3605 Inputs[Idx2], Mask);
3609 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3611 Output = DAG.getVectorShuffle(NewVT,
DL, TmpInputs[Idx1],
3612 TmpInputs[Idx2], Mask);
3616 copy(OrigInputs, std::begin(Inputs));
3621 EVT OVT =
N->getValueType(0);
3629 DAG.getDataLayout().getABITypeAlign(NVT.
getTypeForEVT(*DAG.getContext()));
3631 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV,
Alignment.value());
3632 Hi = DAG.getVAArg(NVT, dl,
Lo.getValue(1), Ptr, SV,
Alignment.value());
3637 ReplaceValueWith(
SDValue(
N, 1), Chain);
3642 EVT DstVTLo, DstVTHi;
3643 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(
N->getValueType(0));
3647 EVT SrcVT =
N->getOperand(0).getValueType();
3649 GetSplitVector(
N->getOperand(0), SrcLo, SrcHi);
3651 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(
N, 0);
3653 Lo = DAG.getNode(
N->getOpcode(), dl, DstVTLo, SrcLo,
N->getOperand(1));
3654 Hi = DAG.getNode(
N->getOpcode(), dl, DstVTHi, SrcHi,
N->getOperand(1));
3659 EVT VT =
N->getValueType(0);
3661 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
3662 assert(LoVT == HiVT &&
"Expected equal split types");
3667 auto [SrcLo, SrcHi] = DAG.SplitVector(Src,
DL);
3671 DAG.getVTList(SplitSrcVT, SplitSrcVT), SrcLo, SrcHi);
3677 DAG.getVTList(LoVT, LoVT), Even, Odd);
3685 GetSplitVector(
N->getOperand(0), InLo, InHi);
3696 SDValue Expanded = TLI.expandVectorSplice(
N, DAG);
3697 std::tie(
Lo,
Hi) = DAG.SplitVector(Expanded,
DL);
3702 EVT VT =
N->getValueType(0);
3725 EVT PtrVT =
StackPtr.getValueType();
3726 auto &MF = DAG.getMachineFunction();
3730 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3733 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3743 DAG.getNode(
ISD::SUB,
DL, PtrVT, DAG.getZExtOrTrunc(EVL,
DL, PtrVT),
3744 DAG.getConstant(1,
DL, PtrVT));
3746 DAG.getConstant(EltWidth,
DL, PtrVT));
3748 SDValue Stride = DAG.getConstant(-(int64_t)EltWidth,
DL, PtrVT);
3750 SDValue TrueMask = DAG.getBoolConstant(
true,
DL,
Mask.getValueType(), VT);
3751 SDValue Store = DAG.getStridedStoreVP(DAG.getEntryNode(),
DL, Val, StorePtr,
3752 DAG.getPOISON(PtrVT), Stride, TrueMask,
3761 std::tie(
Lo,
Hi) = DAG.SplitVector(
Load,
DL);
3766 EVT VT =
N->getValueType(0);
3778 EVL1 = ZExtPromotedInteger(EVL1);
3797 EVT PtrVT =
StackPtr.getValueType();
3798 auto &MF = DAG.getMachineFunction();
3802 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3805 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3811 SDValue EVL1Ptr = DAG.getZExtOrTrunc(DAG.getFreeze(EVL1),
DL, PtrVT);
3816 SDValue StackPtr2 = DAG.getMemBasePlusOffset(StackPtr, EVL1Bytes,
DL);
3817 SDValue PoisonPtr = DAG.getPOISON(PtrVT);
3819 SDValue TrueMask = DAG.getBoolConstant(
true,
DL,
Mask.getValueType(), VT);
3821 DAG.getStoreVP(DAG.getEntryNode(),
DL,
V1, StackPtr, PoisonPtr, TrueMask,
3825 DAG.getStoreVP(StoreV1,
DL, V2, StackPtr2, PoisonPtr, TrueMask, EVL2,
3830 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VT,
N->getOperand(2));
3831 Load = DAG.getLoadVP(VT,
DL, StoreV2, StackPtr, Mask, EVL2, LoadMMO);
3835 SDValue TrailingBytes = DAG.getConstant(TrailingElts * EltWidth,
DL, PtrVT);
3844 Load = DAG.getLoadVP(VT,
DL, StoreV2, StackPtr2, Mask, EVL2, LoadMMO);
3852 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(OrigVT);
3854 DAG.getVectorIdxConstant(0,
DL));
3860void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(
SDNode *
N,
SDValue &
Lo,
3868 GetSplitVector(Acc, AccLo, AccHi);
3869 unsigned Opcode =
N->getOpcode();
3881 GetSplitVector(Input1, Input1Lo, Input1Hi);
3882 GetSplitVector(Input2, Input2Lo, Input2Hi);
3885 Lo = DAG.getNode(Opcode,
DL, ResultVT, AccLo, Input1Lo, Input2Lo);
3886 Hi = DAG.getNode(Opcode,
DL, ResultVT, AccHi, Input1Hi, Input2Hi);
3889void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(
SDNode *
N,
SDValue &
Lo,
3897 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
3908 GetSplitVector(
N->getOperand(0), SourceLo, SourceHi);
3910 GetSplitVector(
N->getOperand(2), MaskLo, MaskHi);
3914 N->getOperand(1), MaskLo,
N->getFlags());
3916 N->getOperand(1), MaskHi,
N->getFlags());
3919void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(
SDNode *
N) {
3920 unsigned Factor =
N->getNumOperands();
3923 for (
unsigned i = 0; i != Factor; ++i) {
3925 GetSplitVector(
N->getOperand(i), OpLo, OpHi);
3927 Ops[i * 2 + 1] = OpHi;
3938 for (
unsigned i = 0; i != Factor; ++i)
3942void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(
SDNode *
N) {
3943 unsigned Factor =
N->getNumOperands();
3946 for (
unsigned i = 0; i != Factor; ++i) {
3948 GetSplitVector(
N->getOperand(i), OpLo, OpHi);
3950 Ops[i + Factor] = OpHi;
3961 for (
unsigned i = 0; i != Factor; ++i) {
3962 unsigned IdxLo = 2 * i;
3963 unsigned IdxHi = 2 * i + 1;
3964 SetSplitVector(
SDValue(
N, i), Res[IdxLo / Factor].
getValue(IdxLo % Factor),
3965 Res[IdxHi / Factor].
getValue(IdxHi % Factor));
3977bool DAGTypeLegalizer::SplitVectorOperand(
SDNode *
N,
unsigned OpNo) {
3982 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
3985 switch (
N->getOpcode()) {
3988 dbgs() <<
"SplitVectorOperand Op #" << OpNo <<
": ";
3997 case ISD::SETCC: Res = SplitVecOp_VSETCC(
N);
break;
4004 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(
N);
4007 Res = SplitVecOp_TruncateHelper(
N);
4013 Res = SplitVecOp_FP_ROUND(
N);
4025 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
4032 case ISD::VP_SCATTER:
4036 case ISD::VP_GATHER:
4040 Res = SplitVecOp_VSELECT(
N, OpNo);
4046 Res = SplitVecOp_MaskedBinOp(
N, OpNo);
4049 Res = SplitVecOp_VECTOR_COMPRESS(
N, OpNo);
4052 Res = SplitVecOp_VECTOR_SHUFFLE_VAR(
N, OpNo);
4058 if (
N->getValueType(0).bitsLT(
4059 N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType()))
4060 Res = SplitVecOp_TruncateHelper(
N);
4062 Res = SplitVecOp_UnaryOp(
N);
4066 Res = SplitVecOp_FP_TO_XINT_SAT(
N);
4082 Res = SplitVecOp_UnaryOp(
N);
4085 Res = SplitVecOp_FPOpDifferentTypes(
N);
4090 Res = SplitVecOp_CMP(
N);
4094 Res = SplitVecOp_FAKE_USE(
N);
4099 Res = SplitVecOp_ExtVecInRegOp(
N);
4119 Res = SplitVecOp_VECREDUCE(
N, OpNo);
4123 Res = SplitVecOp_VECREDUCE_SEQ(
N);
4125 case ISD::VP_REDUCE_FADD:
4126 case ISD::VP_REDUCE_SEQ_FADD:
4127 case ISD::VP_REDUCE_FMUL:
4128 case ISD::VP_REDUCE_SEQ_FMUL:
4129 case ISD::VP_REDUCE_ADD:
4130 case ISD::VP_REDUCE_MUL:
4131 case ISD::VP_REDUCE_AND:
4132 case ISD::VP_REDUCE_OR:
4133 case ISD::VP_REDUCE_XOR:
4134 case ISD::VP_REDUCE_SMAX:
4135 case ISD::VP_REDUCE_SMIN:
4136 case ISD::VP_REDUCE_UMAX:
4137 case ISD::VP_REDUCE_UMIN:
4138 case ISD::VP_REDUCE_FMAX:
4139 case ISD::VP_REDUCE_FMIN:
4140 case ISD::VP_REDUCE_FMAXIMUM:
4141 case ISD::VP_REDUCE_FMINIMUM:
4142 Res = SplitVecOp_VP_REDUCE(
N, OpNo);
4146 Res = SplitVecOp_CttzElts(
N);
4148 case ISD::VP_CTTZ_ELTS:
4149 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4150 Res = SplitVecOp_VP_CttzElements(
N);
4153 Res = SplitVecOp_VECTOR_HISTOGRAM(
N);
4159 Res = SplitVecOp_PARTIAL_REDUCE_MLA(
N);
4162 Res = SplitVecOp_VECTOR_MATCH(
N, OpNo);
4167 if (!Res.
getNode())
return false;
4174 if (
N->isStrictFPOpcode())
4176 "Invalid operand expansion");
4179 "Invalid operand expansion");
4181 ReplaceValueWith(
SDValue(
N, 0), Res);
4185SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(
SDNode *
N) {
4189 GetSplitVector(
N->getOperand(0), LoMask, HiMask);
4191 EVT VT =
N->getValueType(0);
4204 getSetCCResultType(MVT::i1), MVT::i1);
4209 DAG.getElementCount(
DL, VT, SplitEC)),
4213SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(
SDNode *
N,
unsigned OpNo) {
4216 assert(OpNo == 0 &&
"Illegal operand must be mask");
4221 EVT Src0VT =
Src0.getValueType();
4223 assert(
Mask.getValueType().isVector() &&
"VSELECT without a vector mask?");
4226 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
4227 assert(
Lo.getValueType() ==
Hi.getValueType() &&
4228 "Lo and Hi have differing types");
4231 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
4232 assert(LoOpVT == HiOpVT &&
"Asymmetric vector split?");
4235 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0,
DL);
4236 std::tie(LoOp1,
HiOp1) = DAG.SplitVector(Src1,
DL);
4237 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask,
DL);
4247SDValue DAGTypeLegalizer::SplitVecOp_MaskedBinOp(
SDNode *
N,
unsigned OpNo) {
4248 assert(OpNo == 2 &&
"Illegal operand must be mask");
4251 auto [LHSLo, LHSHi] = DAG.SplitVector(
N->getOperand(0),
DL);
4252 auto [RHSLo, RHSHi] = DAG.SplitVector(
N->getOperand(1),
DL);
4254 GetSplitVector(
N->getOperand(2), MaskLo, MaskHi);
4257 RHSLo, MaskLo,
N->getFlags());
4259 RHSHi, MaskHi,
N->getFlags());
4263SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(
SDNode *
N,
unsigned OpNo) {
4266 assert(OpNo == 1 &&
"Illegal operand must be mask");
4271 SplitVecRes_VECTOR_COMPRESS(
N,
Lo,
Hi);
4273 EVT VecVT =
N->getValueType(0);
4277SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_SHUFFLE_VAR(
SDNode *
N,
4281 assert(OpNo == 1 &&
"Illegal operand must be mask");
4283 return TLI.expandVECTOR_SHUFFLE_VAR(
N, DAG);
4286SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(
SDNode *
N,
unsigned OpNo) {
4287 EVT ResVT =
N->getValueType(0);
4293 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
4294 GetSplitVector(VecOp,
Lo,
Hi);
4296 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
4301 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT,
Lo,
Hi,
N->getFlags());
4302 return DAG.getNode(
N->getOpcode(), dl, ResVT, Partial,
N->getFlags());
4306 EVT ResVT =
N->getValueType(0);
4312 SDNodeFlags
Flags =
N->getFlags();
4315 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
4316 GetSplitVector(VecOp,
Lo,
Hi);
4318 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
4324 return DAG.getNode(
N->getOpcode(), dl, ResVT, Partial,
Hi, Flags);
4327SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
4328 assert(
N->isVPOpcode() &&
"Expected VP opcode");
4329 assert(OpNo == 1 &&
"Can only split reduce vector operand");
4331 unsigned Opc =
N->getOpcode();
4332 EVT ResVT =
N->getValueType(0);
4338 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
4339 GetSplitVector(VecOp,
Lo,
Hi);
4342 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(2));
4345 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(
N->getOperand(3), VecVT, dl);
4347 const SDNodeFlags
Flags =
N->getFlags();
4351 return DAG.getNode(
Opc, dl, ResVT, {ResLo,
Hi, MaskHi, EVLHi},
Flags);
4356 EVT ResVT =
N->getValueType(0);
4359 GetSplitVector(
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0),
Lo,
Hi);
4360 EVT InVT =
Lo.getValueType();
4365 if (
N->isStrictFPOpcode()) {
4366 Lo = DAG.getNode(
N->getOpcode(), dl, {OutVT, MVT::Other},
4367 {N->getOperand(0), Lo});
4368 Hi = DAG.getNode(
N->getOpcode(), dl, {OutVT, MVT::Other},
4369 {N->getOperand(0), Hi});
4378 ReplaceValueWith(
SDValue(
N, 1), Ch);
4380 Lo = DAG.getNode(
N->getOpcode(), dl, OutVT,
Lo);
4381 Hi = DAG.getNode(
N->getOpcode(), dl, OutVT,
Hi);
4390 GetSplitVector(
N->getOperand(1),
Lo,
Hi);
4400 EVT ResVT =
N->getValueType(0);
4402 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
4406 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(ResVT);
4412 Lo = BitConvertToInteger(
Lo);
4413 Hi = BitConvertToInteger(
Hi);
4415 if (DAG.getDataLayout().isBigEndian())
4423 assert(OpNo == 1 &&
"Invalid OpNo; can only split SubVec.");
4425 EVT ResVT =
N->getValueType(0);
4433 GetSplitVector(SubVec,
Lo,
Hi);
4442 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
4444 return SecondInsertion;
4447SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
4454 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
4456 ElementCount LoElts =
Lo.getValueType().getVectorElementCount();
4458 ElementCount IdxVal =
4462 EVT SrcVT =
N->getOperand(0).getValueType();
4481 DAG.ExtractVectorElements(
Lo, Elts, IdxValMin,
4482 LoEltsMin - IdxValMin);
4483 DAG.ExtractVectorElements(
Hi, Elts, 0,
4486 return DAG.getBuildVector(SubVT, dl, Elts);
4490 ElementCount ExtractIdx = IdxVal - LoElts;
4492 return DAG.getExtractSubvector(dl, SubVT,
Hi,
4495 EVT HiVT =
Hi.getValueType();
4497 "Only fixed-vector extracts are supported in this case");
4507 DAG.getVectorShuffle(HiVT, dl,
Hi, DAG.getPOISON(HiVT), Mask);
4508 return DAG.getExtractSubvector(dl, SubVT, Shuffle, 0);
4514 "Extracting scalable subvector from fixed-width unsupported");
4522 "subvector from a scalable predicate vector");
4528 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
4530 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
4531 auto &MF = DAG.getMachineFunction();
4535 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
4539 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
4542 SubVT, dl,
Store, StackPtr,
4546SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
4555 GetSplitVector(Vec,
Lo,
Hi);
4557 uint64_t LoElts =
Lo.getValueType().getVectorMinNumElements();
4559 if (IdxVal < LoElts)
4560 return SDValue(DAG.UpdateNodeOperands(
N,
Lo, Idx), 0);
4563 DAG.getConstant(IdxVal - LoElts, SDLoc(
N),
4564 Idx.getValueType())), 0);
4568 if (CustomLowerNode(
N,
N->getValueType(0),
true))
4580 return DAG.getAnyExtOrTrunc(NewExtract, dl,
N->getValueType(0));
4586 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
4588 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
4589 auto &MF = DAG.getMachineFunction();
4592 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
4596 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
4600 assert(
N->getValueType(0).bitsGE(EltVT) &&
"Illegal EXTRACT_VECTOR_ELT.");
4602 return DAG.getExtLoad(
4613 SplitVecRes_ExtVecInRegOp(
N,
Lo,
Hi);
4621 SplitVecRes_Gather(
N,
Lo,
Hi);
4624 ReplaceValueWith(
SDValue(
N, 0), Res);
4629 assert(
N->isUnindexed() &&
"Indexed vp_store of vector?");
4633 assert(
Offset.isUndef() &&
"Unexpected VP store offset");
4635 SDValue EVL =
N->getVectorLength();
4643 GetSplitVector(
Data, DataLo, DataHi);
4645 std::tie(DataLo, DataHi) = DAG.SplitVector(
Data,
DL);
4650 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
4653 GetSplitVector(Mask, MaskLo, MaskHi);
4655 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask,
DL);
4658 EVT MemoryVT =
N->getMemoryVT();
4659 EVT LoMemVT, HiMemVT;
4660 bool HiIsEmpty =
false;
4661 std::tie(LoMemVT, HiMemVT) =
4662 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.
getValueType(), &HiIsEmpty);
4666 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL,
Data.getValueType(),
DL);
4669 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4672 MMOMetadata(
N->getAAInfo(),
N->getRanges()));
4674 Lo = DAG.getStoreVP(Ch,
DL, DataLo, Ptr,
Offset, MaskLo, EVLLo, LoMemVT, MMO,
4675 N->getAddressingMode(),
N->isTruncatingStore(),
4676 N->isCompressingStore());
4682 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo,
DL, LoMemVT, DAG,
4683 N->isCompressingStore());
4685 MachinePointerInfo MPI;
4689 MPI = MachinePointerInfo(
N->getPointerInfo().getAddrSpace());
4694 MMO = DAG.getMachineFunction().getMachineMemOperand(
4696 Alignment, MMOMetadata(
N->getAAInfo(),
N->getRanges()));
4698 Hi = DAG.getStoreVP(Ch,
DL, DataHi, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
4699 N->getAddressingMode(),
N->isTruncatingStore(),
4700 N->isCompressingStore());
4709 assert(
N->isUnindexed() &&
"Indexed vp_strided_store of a vector?");
4710 assert(
N->getOffset().isUndef() &&
"Unexpected VP strided store offset");
4717 GetSplitVector(
Data, LoData, HiData);
4719 std::tie(LoData, HiData) = DAG.SplitVector(
Data,
DL);
4721 EVT LoMemVT, HiMemVT;
4722 bool HiIsEmpty =
false;
4723 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
4729 SplitVecRes_SETCC(
Mask.getNode(), LoMask, HiMask);
4730 else if (getTypeAction(
Mask.getValueType()) ==
4732 GetSplitVector(Mask, LoMask, HiMask);
4734 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask,
DL);
4737 std::tie(LoEVL, HiEVL) =
4738 DAG.SplitEVL(
N->getVectorLength(),
Data.getValueType(),
DL);
4742 N->getChain(),
DL, LoData,
N->getBasePtr(),
N->getOffset(),
4743 N->getStride(), LoMask, LoEVL, LoMemVT,
N->getMemOperand(),
4744 N->getAddressingMode(),
N->isTruncatingStore(),
N->isCompressingStore());
4755 EVT PtrVT =
N->getBasePtr().getValueType();
4758 DAG.getSExtOrTrunc(
N->getStride(),
DL, PtrVT));
4766 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4767 MachinePointerInfo(
N->getPointerInfo().getAddrSpace()),
4769 Alignment, MMOMetadata(
N->getAAInfo(),
N->getRanges()));
4772 N->getChain(),
DL, HiData, Ptr,
N->getOffset(),
N->getStride(), HiMask,
4773 HiEVL, HiMemVT, MMO,
N->getAddressingMode(),
N->isTruncatingStore(),
4774 N->isCompressingStore());
4783 assert(
N->isUnindexed() &&
"Indexed masked store of vector?");
4787 assert(
Offset.isUndef() &&
"Unexpected indexed masked store offset");
4796 GetSplitVector(
Data, DataLo, DataHi);
4798 std::tie(DataLo, DataHi) = DAG.SplitVector(
Data,
DL);
4803 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
4806 GetSplitVector(Mask, MaskLo, MaskHi);
4808 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask,
DL);
4811 EVT MemoryVT =
N->getMemoryVT();
4812 EVT LoMemVT, HiMemVT;
4813 bool HiIsEmpty =
false;
4814 std::tie(LoMemVT, HiMemVT) =
4815 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.
getValueType(), &HiIsEmpty);
4818 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4821 MMOMetadata(
N->getAAInfo(),
N->getRanges(),
N->getMemCacheHint()));
4823 Lo = DAG.getMaskedStore(Ch,
DL, DataLo, Ptr,
Offset, MaskLo, LoMemVT, MMO,
4824 N->getAddressingMode(),
N->isTruncatingStore(),
4825 N->isCompressingStore());
4833 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo,
DL, LoMemVT, DAG,
4834 N->isCompressingStore());
4836 MachinePointerInfo MPI;
4840 MPI = MachinePointerInfo(
N->getPointerInfo().getAddrSpace());
4845 MMO = DAG.getMachineFunction().getMachineMemOperand(
4848 MMOMetadata(
N->getAAInfo(),
N->getRanges(),
N->getMemCacheHint()));
4850 Hi = DAG.getMaskedStore(Ch,
DL, DataHi, Ptr,
Offset, MaskHi, HiMemVT, MMO,
4851 N->getAddressingMode(),
N->isTruncatingStore(),
4852 N->isCompressingStore());
4865 EVT MemoryVT =
N->getMemoryVT();
4875 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
4879 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
4884 EVT LoMemVT, HiMemVT;
4885 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
4890 GetSplitVector(
Ops.Data, DataLo, DataHi);
4892 std::tie(DataLo, DataHi) = DAG.SplitVector(
Ops.Data,
DL);
4897 SplitVecRes_SETCC(
Ops.Mask.getNode(), MaskLo, MaskHi);
4899 std::tie(MaskLo, MaskHi) = SplitMask(
Ops.Mask,
DL);
4903 if (getTypeAction(
Ops.Index.getValueType()) ==
4905 GetSplitVector(
Ops.Index, IndexLo, IndexHi);
4907 std::tie(IndexLo, IndexHi) = DAG.SplitVector(
Ops.Index,
DL);
4911 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4913 Alignment, MMOMetadata(
N->getAAInfo(),
N->getRanges()));
4916 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo,
Ops.Scale};
4918 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
DL, OpsLo, MMO,
4919 MSC->getIndexType(), MSC->isTruncatingStore());
4924 SDValue OpsHi[] = {
Lo, DataHi, MaskHi, Ptr, IndexHi,
Ops.Scale};
4925 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
DL, OpsHi,
4926 MMO, MSC->getIndexType(),
4927 MSC->isTruncatingStore());
4931 std::tie(EVLLo, EVLHi) =
4932 DAG.SplitEVL(VPSC->getVectorLength(),
Ops.Data.getValueType(),
DL);
4934 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo,
Ops.Scale, MaskLo, EVLLo};
4935 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT,
DL, OpsLo, MMO,
4936 VPSC->getIndexType());
4941 SDValue OpsHi[] = {
Lo, DataHi, Ptr, IndexHi,
Ops.Scale, MaskHi, EVLHi};
4942 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT,
DL, OpsHi, MMO,
4943 VPSC->getIndexType());
4947 assert(
N->isUnindexed() &&
"Indexed store of vector?");
4948 assert(OpNo == 1 &&
"Can only split the stored value");
4951 bool isTruncating =
N->isTruncatingStore();
4954 EVT MemoryVT =
N->getMemoryVT();
4957 MMOMetadata
Metadata =
N->getMMOMetadataForSubAccess();
4959 GetSplitVector(
N->getOperand(1),
Lo,
Hi);
4961 EVT LoMemVT, HiMemVT;
4962 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
4966 return TLI.scalarizeVectorStore(
N, DAG);
4969 Lo = DAG.getTruncStore(Ch,
DL,
Lo, Ptr,
N->getPointerInfo(), LoMemVT,
4972 Lo = DAG.getStore(Ch,
DL,
Lo, Ptr,
N->getPointerInfo(), Alignment, MMOFlags,
4975 MachinePointerInfo MPI;
4976 IncrementPointer(
N, LoMemVT, MPI, Ptr);
4979 Hi = DAG.getTruncStore(Ch,
DL,
Hi, Ptr, MPI, HiMemVT, Alignment, MMOFlags,
4982 Hi = DAG.getStore(Ch,
DL,
Hi, Ptr, MPI, Alignment, MMOFlags,
Metadata);
4989 LLVMContext &Ctx = *DAG.getContext();
5007 EVT WideVT = TLI.getLegalTypeToTransformTo(Ctx, IntVecVT);
5008 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(MemIntVT) &&
5012 SDValue Wide = ModifyToType(DAG.getBitcast(IntVecVT, StVal), WideVT);
5015 SDValue Elt = DAG.getExtractVectorElt(
DL, MemIntVT,
5016 DAG.getBitcast(MemVecVT, Wide), 0);
5018 N->getBasePtr(),
N->getMemOperand());
5026 SDValue AsInt = DAG.getBitcast(IntVT, StVal);
5028 N->getBasePtr(),
N->getMemOperand());
5042 for (
unsigned i = 0, e =
Op.getValueType().getVectorNumElements();
5048 return DAG.getBuildVector(
N->getValueType(0),
DL, Elts);
5069 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
5070 SDValue InVec =
N->getOperand(OpNo);
5072 EVT OutVT =
N->getValueType(0);
5080 EVT LoOutVT, HiOutVT;
5081 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
5082 assert(LoOutVT == HiOutVT &&
"Unequal split?");
5087 if (isTypeLegal(LoOutVT) || InElementSize <= OutElementSize * 2 ||
5089 return SplitVecOp_UnaryOp(
N);
5098 return SplitVecOp_UnaryOp(
N);
5102 GetSplitVector(InVec, InLoVec, InHiVec);
5108 EVT HalfElementVT = IsFloat ?
5110 EVT::getIntegerVT(*DAG.
getContext(), InElementSize/2);
5117 if (
N->isStrictFPOpcode()) {
5118 HalfLo = DAG.
getNode(
N->getOpcode(),
DL, {HalfVT, MVT::Other},
5119 {N->getOperand(0), InLoVec});
5120 HalfHi = DAG.
getNode(
N->getOpcode(),
DL, {HalfVT, MVT::Other},
5121 {N->getOperand(0), InHiVec});
5127 HalfLo = DAG.
getNode(
N->getOpcode(),
DL, HalfVT, InLoVec);
5128 HalfHi = DAG.
getNode(
N->getOpcode(),
DL, HalfVT, InHiVec);
5132 EVT InterVT =
EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
5140 if (
N->isStrictFPOpcode()) {
5144 DAG.getTargetConstant(0,
DL, TLI.getPointerTy(DAG.getDataLayout()))});
5152 DAG.getTargetConstant(
5153 0,
DL, TLI.getPointerTy(DAG.getDataLayout())))
5160 assert(
N->getValueType(0).isVector() &&
5161 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
5162 "Operand types must be vectors");
5164 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
5166 GetSplitVector(
N->getOperand(isStrict ? 1 : 0), Lo0, Hi0);
5167 GetSplitVector(
N->getOperand(isStrict ? 2 : 1), Lo1, Hi1);
5169 EVT VT =
N->getValueType(0);
5170 EVT PartResVT = getSetCCResultType(Lo0.
getValueType());
5176 assert(isStrict &&
"unexpected node");
5177 LoRes = DAG.
getNode(
Opc,
DL, DAG.getVTList(PartResVT,
N->getValueType(1)),
5178 N->getOperand(0), Lo0, Lo1,
N->getOperand(3));
5179 HiRes = DAG.
getNode(
Opc,
DL, DAG.getVTList(PartResVT,
N->getValueType(1)),
5180 N->getOperand(0), Hi0, Hi1,
N->getOperand(3));
5183 ReplaceValueWith(
SDValue(
N, 1), NewChain);
5191 EVT OpVT =
N->getOperand(0).getValueType();
5194 return DAG.getExtOrTrunc(Con,
DL, VT, ExtendCode);
5200 EVT ResVT =
N->getValueType(0);
5203 GetSplitVector(
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0),
Lo,
Hi);
5204 EVT InVT =
Lo.getValueType();
5209 if (
N->isStrictFPOpcode()) {
5210 Lo = DAG.getNode(
N->getOpcode(),
DL, {OutVT, MVT::Other},
5211 {N->getOperand(0), Lo, N->getOperand(2)});
5212 Hi = DAG.getNode(
N->getOpcode(),
DL, {OutVT, MVT::Other},
5213 {N->getOperand(0), Hi, N->getOperand(2)});
5217 Lo.getValue(1),
Hi.getValue(1));
5218 ReplaceValueWith(
SDValue(
N, 1), NewChain);
5220 Lo = DAG.getNode(
N->getOpcode(),
DL, OutVT,
Lo,
N->getOperand(1),
5221 N->getOperand(2),
N->getOperand(3));
5222 Hi = DAG.getNode(
N->getOpcode(),
DL, OutVT,
Hi,
N->getOperand(1),
5223 N->getOperand(2),
N->getOperand(3));
5225 Lo = DAG.getNode(
N->getOpcode(),
DL, OutVT,
Lo,
N->getOperand(1));
5226 Hi = DAG.getNode(
N->getOpcode(),
DL, OutVT,
Hi,
N->getOperand(1));
5237SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(
SDNode *
N) {
5240 EVT LHSLoVT, LHSHiVT;
5241 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
5243 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
5244 return DAG.UnrollVectorOp(
N,
N->getValueType(0).getVectorNumElements());
5247 std::tie(LHSLo, LHSHi) =
5248 DAG.SplitVector(
N->getOperand(0),
DL, LHSLoVT, LHSHiVT);
5251 std::tie(RHSLo, RHSHi) = DAG.SplitVector(
N->getOperand(1),
DL);
5254 SDValue Hi = DAG.getNode(
N->getOpcode(),
DL, LHSHiVT, LHSHi, RHSHi);
5260 LLVMContext &Ctxt = *DAG.getContext();
5263 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5264 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
5265 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
5267 EVT ResVT =
N->getValueType(0);
5272 SDValue Lo = DAG.getNode(
N->getOpcode(), dl, NewResVT, LHSLo, RHSLo);
5273 SDValue Hi = DAG.getNode(
N->getOpcode(), dl, NewResVT, LHSHi, RHSHi);
5279 EVT ResVT =
N->getValueType(0);
5282 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
5283 EVT InVT =
Lo.getValueType();
5289 Lo = DAG.getNode(
N->getOpcode(), dl, NewResVT,
Lo,
N->getOperand(1));
5290 Hi = DAG.getNode(
N->getOpcode(), dl, NewResVT,
Hi,
N->getOperand(1));
5297 EVT ResVT =
N->getValueType(0);
5301 GetSplitVector(VecOp,
Lo,
Hi);
5307 DAG.getElementCount(
DL, ResVT,
Lo.getValueType().getVectorElementCount());
5309 DAG.getSetCC(
DL, getSetCCResultType(ResVT), ResLo, VL,
ISD::SETNE);
5311 return DAG.getSelect(
DL, ResVT, ResLoNotVL, ResLo,
5312 DAG.getNode(
ISD::ADD,
DL, ResVT, VL, ResHi));
5317 EVT ResVT =
N->getValueType(0);
5321 GetSplitVector(VecOp,
Lo,
Hi);
5323 auto [MaskLo, MaskHi] = SplitMask(
N->getOperand(1));
5324 auto [EVLLo, EVLHi] =
5326 SDValue VLo = DAG.getZExtOrTrunc(EVLLo,
DL, ResVT);
5332 DAG.getSetCC(
DL, getSetCCResultType(ResVT), ResLo, VLo,
ISD::SETNE);
5334 return DAG.getSelect(
DL, ResVT, ResLoNotEVL, ResLo,
5335 DAG.getNode(
ISD::ADD,
DL, ResVT, VLo, ResHi));
5338SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(
SDNode *
N) {
5349 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5350 std::tie(IndexLo, IndexHi) = DAG.SplitVector(HG->
getIndex(),
DL);
5351 std::tie(MaskLo, MaskHi) = DAG.SplitVector(HG->
getMask(),
DL);
5352 SDValue OpsLo[] = {HG->
getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5353 SDValue Lo = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT,
DL,
5354 OpsLo, MMO, IndexType);
5355 SDValue OpsHi[] = {
Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5356 return DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT,
DL, OpsHi,
5360SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(
SDNode *
N,
unsigned OpNo) {
5364 EVT LoResVT, HiResVT;
5365 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
5367 std::tie(SourceLo, SourceHi) = DAG.SplitVectorOperand(
N, 0);
5369 std::tie(MaskLo, MaskHi) = DAG.SplitVectorOperand(
N, 2);
5372 N->getOperand(1), MaskLo,
N->getFlags());
5374 N->getOperand(1), MaskHi,
N->getFlags());
5380 assert(OpNo == 1 &&
"Unexpected VECTOR_MATCH operand");
5383 GetSplitVector(
N->getOperand(1), NeedleLo, NeedleHi);
5387 NeedleLo,
N->getOperand(2),
N->getFlags());
5390 NeedleHi,
N->getOperand(2),
N->getFlags());
5391 return DAG.getNode(
ISD::OR,
DL,
N->getValueType(0), MatchLo, MatchHi);
5394SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(
SDNode *
N) {
5397 "Accumulator should already be a legal type, and shouldn't need "
5398 "further splitting");
5401 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5402 GetSplitVector(
N->getOperand(1), Input1Lo, Input1Hi);
5403 GetSplitVector(
N->getOperand(2), Input2Lo, Input2Hi);
5404 unsigned Opcode =
N->getOpcode();
5407 SDValue Lo = DAG.getNode(Opcode,
DL, ResultVT, Acc, Input1Lo, Input2Lo);
5408 return DAG.getNode(Opcode,
DL, ResultVT,
Lo, Input1Hi, Input2Hi);
5415void DAGTypeLegalizer::ReplaceOtherWidenResults(
SDNode *
N,
SDNode *WidenNode,
5416 unsigned WidenResNo) {
5417 unsigned NumResults =
N->getNumValues();
5418 for (
unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5419 if (ResNo == WidenResNo)
5421 EVT ResVT =
N->getValueType(ResNo);
5427 DAG.getExtractSubvector(
DL, ResVT,
SDValue(WidenNode, ResNo), 0);
5428 ReplaceValueWith(
SDValue(
N, ResNo), ResVal);
5433void DAGTypeLegalizer::WidenVectorResult(
SDNode *
N,
unsigned ResNo) {
5434 LLVM_DEBUG(
dbgs() <<
"Widen node result " << ResNo <<
": ";
N->dump(&DAG));
5437 if (CustomWidenLowerNode(
N,
N->getValueType(ResNo)))
5442 auto unrollExpandedOp = [&]() {
5450 EVT ResVT =
N->getValueType(ResNo);
5451 EVT WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
5455 EVT VT0 =
N->getValueType(0);
5459 if (!TLI.isOperationLegalOrCustomOrPromote(
N->getOpcode(), WideVecVT) &&
5460 TLI.isOperationExpandOrLibCall(
N->getOpcode(), VT0.
getScalarType())) {
5464 if (
N->getNumValues() > 1)
5465 ReplaceOtherWidenResults(
N, Unrolled.
getNode(), ResNo);
5471 switch (
N->getOpcode()) {
5474 dbgs() <<
"WidenVectorResult #" << ResNo <<
": ";
5482 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(
N);
5486 Res = WidenVecRes_ADDRSPACECAST(
N);
5493 Res = WidenVecRes_INSERT_SUBVECTOR(
N);
5500 case ISD::LOAD: Res = WidenVecRes_LOAD(
N);
break;
5504 Res = WidenVecRes_ScalarOp(
N);
5510 Res = WidenVecRes_Select(
N);
5513 case ISD::SETCC: Res = WidenVecRes_SETCC(
N);
break;
5515 case ISD::UNDEF: Res = WidenVecRes_UNDEF(
N);
break;
5522 case ISD::VP_LOAD_FF:
5525 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5529 Res = WidenVecRes_VECTOR_COMPRESS(
N);
5532 Res = WidenVecRes_VECTOR_SHUFFLE_VAR(
N);
5540 case ISD::VP_GATHER:
5544 Res = WidenVecRes_VECTOR_REVERSE(
N);
5547 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(
N);
5550 WidenVecRes_VECTOR_INTERLEAVE(
N);
5553 Res = WidenVecRes_VECTOR_MATCH(
N);
5556 WidenVecRes_VECTOR_DEINTERLEAVE(
N);
5612 Res = WidenVecRes_Binary(
N);
5619 Res = WidenVecRes_MaskedBinary(
N);
5624 Res = WidenVecRes_CMP(
N);
5630 if (unrollExpandedOp())
5645 Res = WidenVecRes_BinaryCanTrap(
N);
5654 Res = WidenVecRes_BinaryWithExtraScalarOp(
N);
5657#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5658 case ISD::STRICT_##DAGN:
5659#include "llvm/IR/ConstrainedOps.def"
5660 Res = WidenVecRes_StrictFP(
N);
5669 Res = WidenVecRes_OverflowOp(
N, ResNo);
5673 Res = WidenVecRes_FCOPYSIGN(
N);
5678 Res = WidenVecRes_UnarySameEltsWithScalarArg(
N);
5683 if (!unrollExpandedOp())
5684 Res = WidenVecRes_ExpOp(
N);
5690 Res = WidenVecRes_EXTEND_VECTOR_INREG(
N);
5705 Res = WidenVecRes_Convert(
N);
5710 Res = WidenVecRes_FP_TO_XINT_SAT(
N);
5717 Res = WidenVecRes_XROUND(
N);
5743 if (unrollExpandedOp())
5766 Res = WidenVecRes_Unary(
N);
5771 Res = WidenVecRes_Ternary(
N);
5777 if (!unrollExpandedOp())
5778 Res = WidenVecRes_UnaryOpWithTwoResults(
N, ResNo);
5785 Res = WidenVecRes_PARTIAL_REDUCE_MLA(
N);
5791 SetWidenedVector(
SDValue(
N, ResNo), Res);
5797 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5798 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
5799 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
5800 SDValue InOp3 = GetWidenedVector(
N->getOperand(2));
5801 return DAG.getNode(
N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
5807 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5808 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
5809 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
5810 if (
N->getNumOperands() == 2)
5811 return DAG.getNode(
N->getOpcode(), dl, WidenVT, InOp1, InOp2,
5814 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
5815 assert((
N->getOpcode() == ISD::VP_UDIV ||
N->getOpcode() == ISD::VP_SDIV ||
5816 N->getOpcode() == ISD::VP_UREM ||
N->getOpcode() == ISD::VP_SREM) &&
5817 "Expected VP opcode");
5821 return DAG.getNode(
N->getOpcode(), dl, WidenVT,
5822 {InOp1, InOp2, Mask, N->getOperand(3)},
N->getFlags());
5827 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5828 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
5829 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
5832 *DAG.getContext(),
Mask.getValueType().getVectorElementType());
5833 Mask = ModifyToType(Mask, WideMaskVT,
true);
5834 return DAG.getNode(
N->getOpcode(), dl, WidenVT, InOp1, InOp2, Mask,
5839 LLVMContext &Ctxt = *DAG.getContext();
5844 EVT OpVT =
LHS.getValueType();
5846 LHS = GetWidenedVector(
LHS);
5847 RHS = GetWidenedVector(
RHS);
5848 OpVT =
LHS.getValueType();
5851 EVT WidenResVT = TLI.getTypeToTransformTo(Ctxt,
N->getValueType(0));
5854 return DAG.getNode(
N->getOpcode(), dl, WidenResVT,
LHS,
RHS);
5860SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(
SDNode *
N) {
5863 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5864 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
5865 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
5867 return DAG.
getNode(
N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
5876 unsigned ConcatEnd,
EVT VT,
EVT MaxVT,
5879 if (ConcatEnd == 1) {
5880 VT = ConcatOps[0].getValueType();
5882 return ConcatOps[0];
5885 SDLoc dl(ConcatOps[0]);
5892 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5893 int Idx = ConcatEnd - 1;
5894 VT = ConcatOps[Idx--].getValueType();
5895 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5903 }
while (!
TLI.isTypeLegal(NextVT));
5908 unsigned NumToInsert = ConcatEnd - Idx - 1;
5909 for (
unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5911 ConcatOps[Idx+1] = VecOp;
5912 ConcatEnd = Idx + 2;
5918 unsigned RealVals = ConcatEnd - Idx - 1;
5919 unsigned SubConcatEnd = 0;
5920 unsigned SubConcatIdx = Idx + 1;
5921 while (SubConcatEnd < RealVals)
5922 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5923 while (SubConcatEnd < OpsToConcat)
5924 SubConcatOps[SubConcatEnd++] = undefVec;
5926 NextVT, SubConcatOps);
5927 ConcatEnd = SubConcatIdx + 1;
5932 if (ConcatEnd == 1) {
5933 VT = ConcatOps[0].getValueType();
5935 return ConcatOps[0];
5941 if (
NumOps != ConcatEnd ) {
5943 for (
unsigned j = ConcatEnd; j <
NumOps; ++j)
5944 ConcatOps[j] = UndefVal;
5952 unsigned Opcode =
N->getOpcode();
5954 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5957 const SDNodeFlags
Flags =
N->getFlags();
5963 bool FoundLegalVectorType = TLI.isTypeLegal(VT) && !NumElts.
isScalar();
5964 if (FoundLegalVectorType && DAG.isSafeToSpeculativelyExecute(Opcode)) {
5966 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
5967 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
5968 return DAG.getNode(Opcode, dl, WidenVT, InOp1, InOp2, Flags);
5976 VPOpcode && TLI.isOperationLegalOrCustom(*VPOpcode, WidenVT)) {
5979 TLI.isTypeLegal(WideMaskVT)) {
5980 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
5981 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
5982 SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
5984 DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
5985 N->getValueType(0).getVectorElementCount());
5986 return DAG.
getNode(*VPOpcode, dl, WidenVT, InOp1, InOp2, Mask, EVL,
5995 if (!TLI.isTypeLegal(VT))
6001 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
6002 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
6003 ElementCount CurNumElts =
N->getValueType(0).getVectorElementCount();
6006 unsigned ConcatEnd = 0;
6016 SDValue EOp1 = DAG.getExtractSubvector(dl, VT, InOp1, Idx);
6017 SDValue EOp2 = DAG.getExtractSubvector(dl, VT, InOp2, Idx);
6018 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
6021 CurNumElts -= NumElts;
6036 SDValue EOp1 = DAG.getExtractVectorElt(dl, EltVT, InOp1, Idx);
6037 SDValue EOp2 = DAG.getExtractVectorElt(dl, EltVT, InOp2, Idx);
6038 ConcatOps[ConcatEnd++] =
6039 DAG.
getNode(Opcode, dl, EltVT, EOp1, EOp2, Flags);
6045 if (!TLI.isTypeLegal(VT))
6053 switch (
N->getOpcode()) {
6056 return WidenVecRes_STRICT_FSETCC(
N);
6063 return WidenVecRes_Convert_StrictFP(
N);
6070 unsigned Opcode =
N->getOpcode();
6072 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6076 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
6077 NumElts = NumElts / 2;
6088 unsigned CurNumElts =
N->getValueType(0).getVectorNumElements();
6092 unsigned ConcatEnd = 0;
6099 for (
unsigned i = 1; i < NumOpers; ++i) {
6105 Oper = GetWidenedVector(Oper);
6111 DAG.getPOISON(WideOpVT), Oper,
6112 DAG.getVectorIdxConstant(0, dl));
6124 while (CurNumElts != 0) {
6125 while (CurNumElts >= NumElts) {
6128 for (
unsigned i = 0; i < NumOpers; ++i) {
6131 EVT OpVT =
Op.getValueType();
6136 Op = DAG.getExtractSubvector(dl, OpExtractVT,
Op, Idx);
6142 EVT OperVT[] = {VT, MVT::Other};
6144 ConcatOps[ConcatEnd++] = Oper;
6147 CurNumElts -= NumElts;
6150 NumElts = NumElts / 2;
6152 }
while (!TLI.isTypeLegal(VT) && NumElts != 1);
6155 for (
unsigned i = 0; i != CurNumElts; ++i, ++
Idx) {
6158 for (
unsigned i = 0; i < NumOpers; ++i) {
6161 EVT OpVT =
Op.getValueType();
6169 EVT WidenVT[] = {WidenEltVT, MVT::Other};
6171 ConcatOps[ConcatEnd++] = Oper;
6180 if (Chains.
size() == 1)
6181 NewChain = Chains[0];
6184 ReplaceValueWith(
SDValue(
N, 1), NewChain);
6189SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(
SDNode *
N,
unsigned ResNo) {
6191 EVT ResVT =
N->getValueType(0);
6192 EVT OvVT =
N->getValueType(1);
6193 EVT WideResVT, WideOvVT;
6198 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
6203 WideLHS = GetWidenedVector(
N->getOperand(0));
6204 WideRHS = GetWidenedVector(
N->getOperand(1));
6206 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
6215 N->getOperand(0),
Zero);
6217 N->getOperand(1),
Zero);
6220 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
6221 SDNode *WideNode = DAG.getNode(
6222 N->getOpcode(),
DL, WideVTs, WideLHS, WideRHS).getNode();
6225 unsigned OtherNo = 1 - ResNo;
6226 EVT OtherVT =
N->getValueType(OtherNo);
6233 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
6236 return SDValue(WideNode, ResNo);
6240 LLVMContext &Ctx = *DAG.getContext();
6244 EVT WidenVT = TLI.getTypeToTransformTo(Ctx,
N->getValueType(0));
6249 unsigned Opcode =
N->getOpcode();
6250 const SDNodeFlags
Flags =
N->getFlags();
6256 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
6258 InOp = ZExtPromotedInteger(InOp);
6270 if (
N->getNumOperands() == 1)
6271 return DAG.getNode(Opcode,
DL, VT,
Op, Flags);
6273 return DAG.getNode(Opcode,
DL, VT,
Op,
N->getOperand(1),
N->getOperand(2),
6274 N->getOperand(3), Flags);
6275 return DAG.getNode(Opcode,
DL, VT,
Op,
N->getOperand(1), Flags);
6279 InOp = GetWidenedVector(
N->getOperand(0));
6282 if (InVTEC == WidenEC)
6283 return MakeConvertNode(WidenVT, InOp);
6308 return DAG.getInsertSubvector(
DL, DAG.getPOISON(WidenVT), MidRes, 0);
6312 if (TLI.isTypeLegal(InWidenVT)) {
6320 unsigned NumConcat =
6325 return MakeConvertNode(WidenVT, InVec);
6329 SDValue InVal = DAG.getExtractSubvector(
DL, InWidenVT, InOp, 0);
6331 return MakeConvertNode(WidenVT, InVal);
6340 unsigned MinElts =
N->getValueType(0).getVectorNumElements();
6341 for (
unsigned i=0; i < MinElts; ++i) {
6342 SDValue Val = DAG.getExtractVectorElt(
DL, InEltVT, InOp, i);
6343 Ops[i] = MakeConvertNode(EltVT, Val);
6346 return DAG.getBuildVector(WidenVT,
DL,
Ops);
6351 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6355 EVT SrcVT = Src.getValueType();
6359 Src = GetWidenedVector(Src);
6360 SrcVT = Src.getValueType();
6367 return DAG.getNode(
N->getOpcode(), dl, WidenVT, Src,
N->getOperand(1));
6372 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6376 EVT SrcVT = Src.getValueType();
6380 Src = GetWidenedVector(Src);
6381 SrcVT = Src.getValueType();
6388 return DAG.getNode(
N->getOpcode(), dl, WidenVT, Src);
6391SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(
SDNode *
N) {
6396 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6402 unsigned Opcode =
N->getOpcode();
6408 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
6413 unsigned MinElts =
N->getValueType(0).getVectorNumElements();
6414 for (
unsigned i=0; i < MinElts; ++i) {
6415 NewOps[1] = DAG.getExtractVectorElt(
DL, InEltVT, InOp, i);
6416 Ops[i] = DAG.getNode(Opcode,
DL, EltVTs, NewOps);
6420 ReplaceValueWith(
SDValue(
N, 1), NewChain);
6422 return DAG.getBuildVector(WidenVT,
DL,
Ops);
6425SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
6426 unsigned Opcode =
N->getOpcode();
6430 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6439 InOp = GetWidenedVector(InOp);
6446 return DAG.getNode(Opcode,
DL, WidenVT, InOp);
6453 for (
unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i !=
e; ++i) {
6454 SDValue Val = DAG.getExtractVectorElt(
DL, InSVT, InOp, i);
6471 while (
Ops.size() != WidenNumElts)
6472 Ops.push_back(DAG.getPOISON(WidenSVT));
6474 return DAG.getBuildVector(WidenVT,
DL,
Ops);
6480 if (
N->getOperand(0).getValueType() ==
N->getOperand(1).getValueType())
6481 return WidenVecRes_Binary(
N);
6484 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6491SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(
SDNode *
N) {
6493 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6496 SDValue Arg = GetWidenedVector(FpValue);
6497 return DAG.getNode(
N->getOpcode(), SDLoc(
N), WidenVT, {Arg,
N->
getOperand(1)},
6502 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6503 SDValue InOp = GetWidenedVector(
N->getOperand(0));
6505 EVT ExpVT =
RHS.getValueType();
6510 if (WideExpVT == TLI.getTypeToTransformTo(*DAG.getContext(), ExpVT))
6511 ExpOp = GetWidenedVector(
RHS);
6513 ExpOp = ModifyToType(
RHS, WideExpVT);
6516 return DAG.getNode(
N->getOpcode(), SDLoc(
N), WidenVT, InOp, ExpOp);
6521 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6522 SDValue InOp = GetWidenedVector(
N->getOperand(0));
6523 if (
N->getNumOperands() == 1)
6524 return DAG.getNode(
N->getOpcode(), SDLoc(
N), WidenVT, InOp,
N->getFlags());
6526 return DAG.getNode(
N->getOpcode(), SDLoc(
N), WidenVT, InOp,
N->getOperand(1),
6531 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6536 SDValue WidenLHS = GetWidenedVector(
N->getOperand(0));
6537 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
6538 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
6541SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(
SDNode *
N,
6543 EVT VT0 =
N->getValueType(0);
6544 EVT VT1 =
N->getValueType(1);
6548 "expected both results to be vectors of matching element count");
6550 LLVMContext &Ctx = *DAG.getContext();
6551 SDValue InOp = GetWidenedVector(
N->getOperand(0));
6553 EVT WidenVT = TLI.getTypeToTransformTo(Ctx,
N->getValueType(ResNo));
6560 DAG.getNode(
N->getOpcode(), SDLoc(
N), {WidenVT0, WidenVT1}, InOp)
6563 ReplaceOtherWidenResults(
N, WidenNode, ResNo);
6564 return SDValue(WidenNode, ResNo);
6567SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(
SDNode *
N,
unsigned ResNo) {
6568 SDValue WidenVec = DisintegrateMERGE_VALUES(
N, ResNo);
6569 return GetWidenedVector(WidenVec);
6574 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6586 InOp = GetWidenedVector(InOp);
6590 InOp = DAG.getInsertSubvector(
DL, DAG.getPOISON(InWidenVT), InOp, 0);
6593 return DAG.getAddrSpaceCast(
6594 DL, WidenVT, InOp, AddrSpaceCastN->getSrcAddressSpace(),
6595 AddrSpaceCastN->getDestAddressSpace(), AddrSpaceCastN->getFlags());
6601 EVT VT =
N->getValueType(0);
6602 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6605 switch (getTypeAction(InVT)) {
6619 SDValue NInOp = GetPromotedInteger(InOp);
6621 if (WidenVT.
bitsEq(NInVT)) {
6624 if (DAG.getDataLayout().isBigEndian()) {
6627 DAG.getShiftAmountConstant(ShiftAmt, NInVT, dl));
6645 InOp = GetWidenedVector(InOp);
6647 if (WidenVT.
bitsEq(InVT))
6657 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6662 unsigned NewNumParts = WidenSize / InSize;
6675 EVT OrigInVT =
N->getOperand(0).getValueType();
6680 if (TLI.isTypeLegal(NewInVT)) {
6688 if (WidenSize % InSize == 0) {
6695 DAG.ExtractVectorElements(InOp,
Ops);
6696 Ops.append(WidenSize / InScalarSize -
Ops.size(),
6708 return CreateStackStoreLoad(InOp, WidenVT);
6711SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(
SDNode *
N) {
6713 N->getOpcode(), SDLoc(
N),
6714 TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0)),
6715 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
6721 EVT VT =
N->getValueType(0);
6725 EVT EltVT =
N->getOperand(0).getValueType();
6728 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6732 assert(WidenNumElts >= NumElts &&
"Shrinking vector instead of widening!");
6733 NewOps.append(WidenNumElts - NumElts, DAG.getPOISON(EltVT));
6735 return DAG.getBuildVector(WidenVT, dl, NewOps);
6739 EVT InVT =
N->getOperand(0).getValueType();
6740 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6742 unsigned NumOperands =
N->getNumOperands();
6744 bool InputWidened =
false;
6748 if (WidenNumElts % NumInElts == 0) {
6750 unsigned NumConcat = WidenNumElts / NumInElts;
6751 SDValue UndefVal = DAG.getPOISON(InVT);
6753 for (
unsigned i=0; i < NumOperands; ++i)
6754 Ops[i] =
N->getOperand(i);
6755 for (
unsigned i = NumOperands; i != NumConcat; ++i)
6760 InputWidened =
true;
6761 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
6764 for (i=1; i < NumOperands; ++i)
6765 if (!
N->getOperand(i).isUndef())
6768 if (i == NumOperands)
6771 return GetWidenedVector(
N->getOperand(0));
6773 if (NumOperands == 2) {
6775 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6780 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6781 for (
unsigned i = 0; i < NumInElts; ++i) {
6783 MaskOps[i + NumInElts] = i + WidenNumElts;
6785 return DAG.getVectorShuffle(WidenVT, dl,
6786 GetWidenedVector(
N->getOperand(0)),
6787 GetWidenedVector(
N->getOperand(1)),
6794 SDValue WideVec = DAG.getPOISON(WidenVT);
6796 for (
unsigned I = 0;
I < NumOperands; ++
I)
6798 DAG.getInsertSubvector(dl, WideVec,
N->getOperand(
I),
I * NumInElts);
6809 for (
unsigned i=0; i < NumOperands; ++i) {
6812 InOp = GetWidenedVector(InOp);
6813 for (
unsigned j = 0;
j < NumInElts; ++
j)
6814 Ops[Idx++] = DAG.getExtractVectorElt(dl, EltVT, InOp, j);
6816 SDValue UndefVal = DAG.getPOISON(EltVT);
6817 for (;
Idx < WidenNumElts; ++
Idx)
6818 Ops[Idx] = UndefVal;
6819 return DAG.getBuildVector(WidenVT, dl,
Ops);
6822SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(
SDNode *
N) {
6823 EVT VT =
N->getValueType(0);
6824 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6825 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
6832SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
6833 EVT VT =
N->getValueType(0);
6835 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6840 auto InOpTypeAction = getTypeAction(InOp.
getValueType());
6842 InOp = GetWidenedVector(InOp);
6848 if (IdxVal == 0 && InVT == WidenVT)
6855 assert(IdxVal % VTNumElts == 0 &&
6856 "Expected Idx to be a multiple of subvector minimum vector length");
6857 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6870 unsigned GCD = std::gcd(VTNumElts, WidenNumElts);
6871 assert((IdxVal % GCD) == 0 &&
"Expected Idx to be a multiple of the broken "
6872 "down type's element count");
6879 for (;
I < VTNumElts / GCD; ++
I)
6881 DAG.getExtractSubvector(dl, PartVT, InOp, IdxVal +
I * GCD));
6882 for (;
I < WidenNumElts / GCD; ++
I)
6904 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, StoreMMO);
6911 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, InVT, VT, Idx);
6912 return DAG.getMaskedLoad(
6913 WidenVT, dl, Ch, StackPtr, DAG.getPOISON(
StackPtr.getValueType()), Mask,
6921 for (i = 0; i < VTNumElts; ++i)
6922 Ops[i] = DAG.getExtractVectorElt(dl, EltVT, InOp, IdxVal + i);
6924 SDValue UndefVal = DAG.getPOISON(EltVT);
6925 for (; i < WidenNumElts; ++i)
6927 return DAG.getBuildVector(WidenVT, dl,
Ops);
6933 TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0)),
true);
6938SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(
SDNode *
N) {
6939 SDValue InOp = GetWidenedVector(
N->getOperand(0));
6942 N->getOperand(1),
N->getOperand(2));
6951 "Load width must be less than or equal to first value type width");
6960 assert(FirstVT == WidenVT &&
"First value type must equal widen value type");
6977 assert(FirstVT == WidenVT &&
"First value type must equal widen value type");
6988 TLI.getTypeToTransformTo(*DAG.getContext(),
LD->getValueType(0));
6989 EVT LdVT =
LD->getMemoryVT();
6998 TypeSize WidthDiff = WidenWidth - LdWidth;
7001 std::optional<EVT> FirstVT =
7002 findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT, 0,
7008 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
7011 Chain, BasePtr,
LD->getMemOperand());
7015 FirstVTWidth, dl, DAG);
7033 if (!
LD->getMemoryVT().isByteSized()) {
7035 std::tie(
Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
7037 ReplaceValueWith(
SDValue(LD, 1), NewChain);
7046 EVT VT =
LD->getValueType(0);
7047 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7048 EVT WideMaskVT = getSetCCResultType(WideVT);
7051 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WideVT) &&
7052 TLI.isTypeLegal(WideMaskVT)) {
7055 SDValue EVL = DAG.getElementCount(
DL, TLI.getVPExplicitVectorLengthTy(),
7059 LD->getChain(),
LD->getBasePtr(),
LD->getOffset(), Mask,
7060 EVL,
LD->getMemoryVT(),
LD->getMemOperand());
7072 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
7074 Result = GenWidenVectorLoads(LdChain, LD);
7081 if (LdChain.
size() == 1)
7082 NewChain = LdChain[0];
7088 ReplaceValueWith(
SDValue(
N, 1), NewChain);
7099 SDValue NewLoad = DAG.getMaskedLoad(
7100 WideVT,
DL,
LD->getChain(),
LD->getBasePtr(),
LD->getOffset(), Mask,
7101 DAG.getPOISON(WideVT),
LD->getMemoryVT(),
LD->getMemOperand(),
7102 LD->getAddressingMode(),
LD->getExtensionType());
7112 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7114 SDValue EVL =
N->getVectorLength();
7121 "Unable to widen binary VP op");
7122 Mask = GetWidenedVector(Mask);
7123 assert(
Mask.getValueType().getVectorElementCount() ==
7124 TLI.getTypeToTransformTo(*DAG.getContext(),
Mask.getValueType())
7125 .getVectorElementCount() &&
7126 "Unable to widen vector load");
7129 DAG.getLoadVP(
N->getAddressingMode(), ExtType, WidenVT, dl,
N->getChain(),
7130 N->getBasePtr(),
N->getOffset(), Mask, EVL,
7131 N->getMemoryVT(),
N->getMemOperand(),
N->isExpandingLoad());
7139 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7141 SDValue EVL =
N->getVectorLength();
7147 "Unable to widen binary VP op");
7148 Mask = GetWidenedVector(Mask);
7149 assert(
Mask.getValueType().getVectorElementCount() ==
7150 TLI.getTypeToTransformTo(*DAG.getContext(),
Mask.getValueType())
7151 .getVectorElementCount() &&
7152 "Unable to widen vector load");
7154 SDValue Res = DAG.getLoadFFVP(WidenVT, dl,
N->getChain(),
N->getBasePtr(),
7155 Mask, EVL,
N->getMemOperand());
7168 "Unable to widen VP strided load");
7169 Mask = GetWidenedVector(Mask);
7171 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7172 assert(
Mask.getValueType().getVectorElementCount() ==
7174 "Data and mask vectors should have the same number of elements");
7176 SDValue Res = DAG.getStridedLoadVP(
7177 N->getAddressingMode(),
N->getExtensionType(), WidenVT,
DL,
N->getChain(),
7178 N->getBasePtr(),
N->getOffset(),
N->getStride(), Mask,
7179 N->getVectorLength(),
N->getMemoryVT(),
N->getMemOperand(),
7180 N->isExpandingLoad());
7188SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(
SDNode *
N) {
7193 TLI.getTypeToTransformTo(*DAG.getContext(), Vec.
getValueType());
7195 Mask.getValueType().getVectorElementType(),
7198 SDValue WideVec = ModifyToType(Vec, WideVecVT);
7199 SDValue WideMask = ModifyToType(Mask, WideMaskVT,
true);
7200 SDValue WidePassthru = ModifyToType(Passthru, WideVecVT);
7202 WideMask, WidePassthru);
7205SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE_VAR(
SDNode *
N) {
7207 SDValue Src = GetWidenedVector(
N->getOperand(0));
7208 EVT WideVT = Src.getValueType();
7211 N->getOperand(1).getValueType().getVectorElementType());
7212 SDValue Mask = ModifyToType(
N->getOperand(1), WideMaskVT);
7217 EVT VT =
N->getValueType(0);
7218 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7220 EVT MaskVT =
Mask.getValueType();
7221 SDValue PassThru = GetWidenedVector(
N->getPassThru());
7230 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WidenVT) &&
7231 TLI.isTypeLegal(WideMaskVT) &&
7237 Mask = DAG.getInsertSubvector(dl, DAG.getPOISON(WideMaskVT), Mask, 0);
7238 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
7242 N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, EVL,
7243 N->getMemoryVT(),
N->getMemOperand());
7247 if (!
N->getPassThru()->isUndef()) {
7251 NewVal = DAG.
getNode(ISD::VP_MERGE, dl, WidenVT,
7252 DAG.getAllOnesConstant(dl, WideMaskVT), NewVal,
7253 DAG.getPOISON(WidenVT), EVL);
7264 Mask = ModifyToType(Mask, WideMaskVT,
true);
7266 SDValue Res = DAG.getMaskedLoad(
7267 WidenVT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask,
7268 PassThru,
N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
7269 ExtType,
N->isExpandingLoad());
7278 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7280 EVT MaskVT =
Mask.getValueType();
7281 SDValue PassThru = GetWidenedVector(
N->getPassThru());
7289 Mask = ModifyToType(Mask, WideMaskVT,
true);
7294 *DAG.getContext(),
Index.getValueType().getScalarType(), WideEC);
7295 Index = ModifyToType(Index, WideIndexVT);
7301 N->getMemoryVT().getScalarType(), WideEC);
7302 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
7303 WideMemVT, dl,
Ops,
N->getMemOperand(),
7304 N->getIndexType(),
N->getExtensionType());
7313 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7321 N->getMemoryVT().getScalarType(), WideEC);
7322 Mask = GetWidenedMask(Mask, WideEC);
7325 Mask,
N->getVectorLength()};
7326 SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT,
7327 dl,
Ops,
N->getMemOperand(),
N->getIndexType());
7336 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7337 return DAG.getNode(
N->getOpcode(), SDLoc(
N), WidenVT,
N->getOperand(0));
7365 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
7366 return N->getOperand(OpNo).getValueType();
7374 N =
N.getOperand(0);
7376 for (
unsigned i = 1; i <
N->getNumOperands(); ++i)
7377 if (!
N->getOperand(i)->isUndef())
7379 N =
N.getOperand(0);
7383 N =
N.getOperand(0);
7385 N =
N.getOperand(0);
7410 { MaskVT, MVT::Other },
Ops);
7411 ReplaceValueWith(InMask.
getValue(1),
Mask.getValue(1));
7417 return adjustMaskToType(Mask, ToMaskVT);
7423 LLVMContext &Ctx = *DAG.getContext();
7424 EVT MaskVT =
Mask.getValueType();
7427 if (MaskScalarBits < ToMaskScalBits) {
7431 }
else if (MaskScalarBits > ToMaskScalBits) {
7437 assert(
Mask->getValueType(0).getScalarSizeInBits() ==
7439 "Mask should have the right element size by now.");
7442 unsigned CurrMaskNumEls =
Mask->getValueType(0).getVectorNumElements();
7444 Mask = DAG.getExtractSubvector(SDLoc(Mask), ToMaskVT, Mask, 0);
7447 EVT SubVT =
Mask->getValueType(0);
7453 assert((
Mask->getValueType(0) == ToMaskVT) &&
7454 "A mask of ToMaskVT should have been produced by now.");
7461EVT DAGTypeLegalizer::unifyMaskTypes(
SDValue &Op0,
bool IsOpLenient0,
7462 SDValue &Op1,
bool IsOpLenient1,
7466 "unifyMaskTypes only handles scalar width differences");
7478 if (IsOpLenient0 != IsOpLenient1) {
7479 SDValue *LenientOp, *NonLenientOp;
7482 NonLenientOp = &Op1;
7485 NonLenientOp = &Op0;
7488 *LenientOp = adjustMaskToType(*LenientOp, OpVT);
7494 unsigned NarrowBits = std::min(Bits0, Bits1);
7495 unsigned WideBits = std::max(Bits0, Bits1);
7497 unsigned IntBits = NarrowBits == WideBits ? NarrowBits
7498 : ToBits >= WideBits ? WideBits
7499 : ToBits <= NarrowBits ? NarrowBits
7503 Op0 = adjustMaskToType(Op0, OpVT);
7504 Op1 = adjustMaskToType(Op1, OpVT);
7508std::pair<SDValue, bool>
7509DAGTypeLegalizer::convertMaskTreeImpl(
SDValue V,
EVT ToVT,
unsigned Depth) {
7541 if (
Depth >= DAG.MaxRecursionDepth)
7550 unsigned Opcode =
V.getOpcode();
7555 return {convertMask(
V, MaskVT, MaskVT),
false};
7563 return {DAG.getConstant(0,
DL, ToVT),
true};
7565 return {DAG.getAllOnesConstant(
DL, ToVT),
true};
7570 auto [Op0, IsLenientOp0] =
7571 convertMaskTreeImpl(
V.getOperand(0), ToVT,
Depth + 1);
7574 auto [Op1, IsLenientOp1] =
7575 convertMaskTreeImpl(
V.getOperand(1), ToVT,
Depth + 1);
7578 EVT OpVT = unifyMaskTypes(Op0, IsLenientOp0, Op1, IsLenientOp1, ToVT);
7579 return {DAG.getNode(Opcode,
DL, OpVT, Op0, Op1),
7580 IsLenientOp0 && IsLenientOp1};
7585 auto [Inner, IsTypeLenient] =
7586 convertMaskTreeImpl(
V.getOperand(0), ToVT,
Depth + 1);
7589 return {DAG.getNode(
ISD::FREEZE,
DL, Inner.getValueType(), Inner),
7596 auto [Op0, IsLenientOp0] =
7597 convertMaskTreeImpl(
V.getOperand(0), ToVT,
Depth + 1);
7600 if (
V.getOperand(1).isUndef()) {
7602 return {DAG.getVectorShuffle(OpVT,
DL, Op0, DAG.getUNDEF(OpVT),
7606 auto [Op1, IsLenientOp1] =
7607 convertMaskTreeImpl(
V.getOperand(1), ToVT,
Depth + 1);
7610 EVT OpVT = unifyMaskTypes(Op0, IsLenientOp0, Op1, IsLenientOp1, ToVT);
7611 return {DAG.getVectorShuffle(OpVT,
DL, Op0, Op1, Shuf->getMask()),
7612 IsLenientOp0 && IsLenientOp1};
7617 auto [Op1, IsLenientOp1] =
7618 convertMaskTreeImpl(
V.getOperand(1), ToVT,
Depth + 1);
7621 auto [Op2, IsLenientOp2] =
7622 convertMaskTreeImpl(
V.getOperand(2), ToVT,
Depth + 1);
7625 EVT OpVT = unifyMaskTypes(Op1, IsLenientOp1, Op2, IsLenientOp2, ToVT);
7635 return {DAG.getNode(Opcode,
DL, OpVT,
Cond, Op1, Op2),
7636 IsLenientOp1 && IsLenientOp2};
7650 *DAG.getContext(),
V.getValueType().getVectorElementCount());
7651 auto [
Result,
_] = convertMaskTreeImpl(
V, MaskTreeVT);
7654 return adjustMaskToType(Result, ToVT);
7662 LLVMContext &Ctx = *DAG.getContext();
7670 EVT CondVT =
Cond->getValueType(0);
7674 EVT VSelVT =
N->getValueType(0);
7686 EVT FinalVT = VSelVT;
7697 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
7698 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
7705 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
7713 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
7716 EVT ToMaskVT = VSelVT;
7721 return convertMaskTree(
Cond, ToMaskVT);
7725 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7730 unsigned Opcode =
N->getOpcode();
7732 if (
SDValue WideCond = WidenVSELECTMask(
N)) {
7733 SDValue InOp1 = GetWidenedVector(
N->getOperand(1));
7734 SDValue InOp2 = GetWidenedVector(
N->getOperand(2));
7736 return DAG.getNode(Opcode, SDLoc(
N), WidenVT, WideCond, InOp1, InOp2);
7742 Cond1 = GetWidenedVector(Cond1);
7750 SDValue SplitSelect = SplitVecOp_VSELECT(
N, 0);
7751 SDValue Res = ModifyToType(SplitSelect, WidenVT);
7756 Cond1 = ModifyToType(Cond1, CondWidenVT);
7759 SDValue InOp1 = GetWidenedVector(
N->getOperand(1));
7760 SDValue InOp2 = GetWidenedVector(
N->getOperand(2));
7762 if (Opcode == ISD::VP_MERGE)
7763 return DAG.getNode(Opcode, SDLoc(
N), WidenVT, Cond1, InOp1, InOp2,
7765 return DAG.getNode(Opcode, SDLoc(
N), WidenVT, Cond1, InOp1, InOp2);
7769 SDValue InOp1 = GetWidenedVector(
N->getOperand(2));
7770 SDValue InOp2 = GetWidenedVector(
N->getOperand(3));
7773 N->getOperand(1), InOp1, InOp2,
N->getOperand(4));
7777 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7778 return DAG.getUNDEF(WidenVT);
7782 EVT VT =
N->getValueType(0);
7785 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7789 SDValue InOp1 = GetWidenedVector(
N->getOperand(0));
7790 SDValue InOp2 = GetWidenedVector(
N->getOperand(1));
7793 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7794 for (
unsigned i = 0; i != NumElts; ++i) {
7795 int Idx =
N->getMaskElt(i);
7796 if (Idx < (
int)NumElts)
7799 NewMask[i] =
Idx - NumElts + WidenNumElts;
7801 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
7805 EVT VT =
N->getValueType(0);
7809 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7810 SDValue OpValue = GetWidenedVector(
N->getOperand(0));
7816 unsigned IdxVal = WidenNumElts - VTNumElts;
7829 unsigned GCD = std::gcd(VTNumElts, WidenNumElts);
7832 assert((IdxVal % GCD) == 0 &&
"Expected Idx to be a multiple of the broken "
7833 "down type's element count");
7836 for (; i < VTNumElts / GCD; ++i)
7838 DAG.getExtractSubvector(dl, PartVT, ReverseVal, IdxVal + i * GCD));
7839 for (; i < WidenNumElts / GCD; ++i)
7847 SmallVector<int, 16>
Mask(WidenNumElts, -1);
7848 std::iota(
Mask.begin(),
Mask.begin() + VTNumElts, IdxVal);
7850 return DAG.getVectorShuffle(WidenVT, dl, ReverseVal, DAG.getPOISON(WidenVT),
7854SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(
SDNode *
N) {
7855 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7859void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(
SDNode *
N) {
7860 EVT VT =
N->getValueType(0);
7863 unsigned Factor =
N->getNumOperands();
7866 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7870 for (
unsigned Idx = 0U;
Idx < Factor; ++
Idx)
7871 WidenOps[Idx] = GetWidenedVector(
N->getOperand(Idx));
7880 for (
unsigned Idx = 0;
Idx != Factor; ++
Idx)
7881 Slices[Idx] = Interleaved.
getValue(Idx);
7885 for (
unsigned Idx = 0U;
Idx < Factor; ++
Idx) {
7886 SDValue Narrow = DAG.getExtractSubvector(
DL, VT, Packed,
7889 DAG.getInsertSubvector(
DL, DAG.getPOISON(WidenVT), Narrow, 0U);
7890 SetWidenedVector(
SDValue(
N, Idx), Wide);
7896 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7897 EVT SourceVT =
N->getOperand(0).getValueType();
7902 SDValue WideSource = DAG.getInsertSubvector(
DL, DAG.getUNDEF(WideSourceVT),
7903 N->getOperand(0), 0);
7904 SDValue WideMask = DAG.getInsertSubvector(
DL, DAG.getConstant(0,
DL, WidenVT),
7905 N->getOperand(2), 0);
7907 N->getOperand(1), WideMask,
N->getFlags());
7910void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(
SDNode *
N) {
7911 EVT VT =
N->getValueType(0);
7914 unsigned Factor =
N->getNumOperands();
7917 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
7927 EVT ConcatVT =
EVT::getVectorVT(*DAG.getContext(), EltVT, OrigEC * Factor);
7929 SDValue PackedWidenVec = DAG.getInsertSubvector(
7930 DL, DAG.getUNDEF(PackedWidenVT), ConcatOp, 0U);
7934 for (
unsigned Idx = 0U;
Idx < Factor; ++
Idx) {
7935 NewOps[
Idx] = DAG.getExtractSubvector(
DL, WidenVT, PackedWidenVec,
7942 for (
unsigned Idx = 0U;
Idx < Factor; ++
Idx)
7947 assert(
N->getValueType(0).isVector() &&
7948 N->getOperand(0).getValueType().isVector() &&
7949 "Operands must be vectors");
7950 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
7963 SDValue SplitVSetCC = SplitVecOp_VSETCC(
N);
7964 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
7971 InOp1 = GetWidenedVector(InOp1);
7972 InOp2 = GetWidenedVector(InOp2);
7975 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(
N));
7986 "Input not widened to expected type!");
7988 return DAG.getNode(
ISD::SETCC, SDLoc(
N), WidenVT, InOp1, InOp2,
7993 assert(
N->getValueType(0).isVector() &&
7994 N->getOperand(1).getValueType().isVector() &&
7995 "Operands must be vectors");
7996 EVT VT =
N->getValueType(0);
7997 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
8007 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
8012 for (
unsigned i = 0; i != NumElts; ++i) {
8013 SDValue LHSElem = DAG.getExtractVectorElt(dl, TmpEltVT,
LHS, i);
8014 SDValue RHSElem = DAG.getExtractVectorElt(dl, TmpEltVT,
RHS, i);
8016 Scalars[i] = DAG.getNode(
N->getOpcode(), dl, {MVT::i1, MVT::Other},
8017 {Chain, LHSElem, RHSElem, CC});
8018 Chains[i] = Scalars[i].getValue(1);
8019 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
8020 DAG.getBoolConstant(
true, dl, EltVT, VT),
8021 DAG.getBoolConstant(
false, dl, EltVT, VT));
8025 ReplaceValueWith(
SDValue(
N, 1), NewChain);
8027 return DAG.getBuildVector(WidenVT, dl, Scalars);
8030SDValue DAGTypeLegalizer::WidenVecRes_PARTIAL_REDUCE_MLA(
SDNode *
N) {
8032 EVT VT =
N->getValueType(0);
8035 SDValue Expanded = TLI.expandPartialReduceMLA(
N, DAG);
8036 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
8037 return DAG.getInsertSubvector(
DL, DAG.getPOISON(WideVT), Expanded, 0);
8043bool DAGTypeLegalizer::WidenVectorOperand(
SDNode *
N,
unsigned OpNo) {
8044 LLVM_DEBUG(
dbgs() <<
"Widen node operand " << OpNo <<
": ";
N->dump(&DAG));
8048 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
8051 switch (
N->getOpcode()) {
8054 dbgs() <<
"WidenVectorOperand op #" << OpNo <<
": ";
8062 Res = WidenVecOp_FAKE_USE(
N);
8066 Res = WidenVecOp_VECTOR_REPEAT(
N);
8071 case ISD::STORE: Res = WidenVecOp_STORE(
N);
break;
8075 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(
N, OpNo);
break;
8076 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8077 Res = WidenVecOp_VP_STRIDED_STORE(
N, OpNo);
8082 Res = WidenVecOp_EXTEND_VECTOR_INREG(
N);
8084 case ISD::MSTORE: Res = WidenVecOp_MSTORE(
N, OpNo);
break;
8085 case ISD::MGATHER: Res = WidenVecOp_MGATHER(
N, OpNo);
break;
8087 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(
N, OpNo);
break;
8088 case ISD::SETCC: Res = WidenVecOp_SETCC(
N);
break;
8098 Res = WidenVecOp_UnrollVectorOp(
N);
8105 Res = WidenVecOp_EXTEND(
N);
8110 Res = WidenVecOp_CMP(
N);
8128 Res = WidenVecOp_Convert(
N);
8133 Res = WidenVecOp_FP_TO_XINT_SAT(
N);
8153 Res = WidenVecOp_VECREDUCE(
N);
8157 Res = WidenVecOp_VECREDUCE_SEQ(
N);
8159 case ISD::VP_REDUCE_FADD:
8160 case ISD::VP_REDUCE_SEQ_FADD:
8161 case ISD::VP_REDUCE_FMUL:
8162 case ISD::VP_REDUCE_SEQ_FMUL:
8163 case ISD::VP_REDUCE_ADD:
8164 case ISD::VP_REDUCE_MUL:
8165 case ISD::VP_REDUCE_AND:
8166 case ISD::VP_REDUCE_OR:
8167 case ISD::VP_REDUCE_XOR:
8168 case ISD::VP_REDUCE_SMAX:
8169 case ISD::VP_REDUCE_SMIN:
8170 case ISD::VP_REDUCE_UMAX:
8171 case ISD::VP_REDUCE_UMIN:
8172 case ISD::VP_REDUCE_FMAX:
8173 case ISD::VP_REDUCE_FMIN:
8174 case ISD::VP_REDUCE_FMAXIMUM:
8175 case ISD::VP_REDUCE_FMINIMUM:
8176 Res = WidenVecOp_VP_REDUCE(
N);
8180 Res = WidenVecOp_CttzElements(
N);
8182 case ISD::VP_CTTZ_ELTS:
8183 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8184 Res = WidenVecOp_VP_CttzElements(
N);
8187 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(
N);
8190 Res = WidenVecOp_VECTOR_SHUFFLE_VAR(
N, OpNo);
8193 Res = WidenVecOp_VECTOR_MATCH(
N, OpNo);
8199 Res = WidenVecOp_PARTIAL_REDUCE_MLA(
N);
8204 if (!Res.
getNode())
return false;
8212 if (
N->isStrictFPOpcode())
8214 "Invalid operand expansion");
8217 "Invalid operand expansion");
8219 ReplaceValueWith(
SDValue(
N, 0), Res);
8225 EVT VT =
N->getValueType(0);
8230 "Unexpected type action");
8231 InOp = GetWidenedVector(InOp);
8234 "Input wasn't widened!");
8242 EVT FixedEltVT = FixedVT.getVectorElementType();
8243 if (TLI.isTypeLegal(FixedVT) &&
8245 FixedEltVT == InEltVT) {
8247 "Not enough elements in the fixed type for the operand!");
8249 "We can't have the same type as we started with!");
8251 InOp = DAG.getInsertSubvector(
DL, DAG.getPOISON(FixedVT), InOp, 0);
8253 InOp = DAG.getExtractSubvector(
DL, FixedVT, InOp, 0);
8262 return WidenVecOp_Convert(
N);
8267 switch (
N->getOpcode()) {
8282 EVT OpVT =
N->getOperand(0).getValueType();
8283 EVT ResVT =
N->getValueType(0);
8290 LHS = DAG.getExtractSubvector(dl, OpVT,
LHS, 0);
8291 RHS = DAG.getExtractSubvector(dl, OpVT,
RHS, 0);
8297 LHS = DAG.getNode(ExtendOpcode, dl, ResVT,
LHS);
8298 RHS = DAG.getNode(ExtendOpcode, dl, ResVT,
RHS);
8300 return DAG.getNode(
N->getOpcode(), dl, ResVT,
LHS,
RHS);
8307 return DAG.UnrollVectorOp(
N);
8312 EVT ResultVT =
N->getValueType(0);
8314 SDValue WideArg = GetWidenedVector(
N->getOperand(0));
8317 EVT WideResultVT = getSetCCResultType(WideArg.
getValueType());
8323 {WideArg,
Test},
N->getFlags());
8329 SDValue CC = DAG.getExtractSubvector(
DL, ResVT, WideNode, 0);
8331 EVT OpVT =
N->getOperand(0).getValueType();
8334 return DAG.getNode(ExtendCode,
DL, ResultVT,
CC);
8339 EVT VT =
N->getValueType(0);
8345 "Unexpected type action");
8346 InOp = GetWidenedVector(InOp);
8348 unsigned Opcode =
N->getOpcode();
8353 return DAG.getNode(Opcode, dl, VT,
Op,
N->getOperand(1),
N->getOperand(2),
8356 return DAG.getNode(Opcode, dl, VT,
Op,
N->getOperand(1));
8357 return DAG.getNode(Opcode, dl, VT,
Op);
8364 if (TLI.isTypeLegal(WideVT) && !
N->isStrictFPOpcode()) {
8366 if (
N->isStrictFPOpcode()) {
8368 Res = DAG.
getNode(Opcode, dl, { WideVT, MVT::Other },
8371 Res = DAG.
getNode(Opcode, dl, { WideVT, MVT::Other },
8372 {
N->getOperand(0), InOp });
8377 Res = MakeConvertNode(WideVT, InOp);
8379 return DAG.getExtractSubvector(dl, VT, Res, 0);
8387 if (
N->isStrictFPOpcode()) {
8390 for (
unsigned i=0; i < NumElts; ++i) {
8391 NewOps[1] = DAG.getExtractVectorElt(dl, InEltVT, InOp, i);
8392 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
8396 ReplaceValueWith(
SDValue(
N, 1), NewChain);
8398 for (
unsigned i = 0; i < NumElts; ++i) {
8399 SDValue Elt = DAG.getExtractVectorElt(dl, InEltVT, InOp, i);
8400 Ops[i] = MakeConvertNode(EltVT, Elt);
8404 return DAG.getBuildVector(VT, dl,
Ops);
8408 EVT DstVT =
N->getValueType(0);
8409 SDValue Src = GetWidenedVector(
N->getOperand(0));
8410 EVT SrcVT = Src.getValueType();
8417 if (TLI.isTypeLegal(WideDstVT)) {
8419 DAG.
getNode(
N->getOpcode(), dl, WideDstVT, Src,
N->getOperand(1));
8422 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
8426 return DAG.UnrollVectorOp(
N);
8430 EVT VT =
N->getValueType(0);
8431 SDValue InOp = GetWidenedVector(
N->getOperand(0));
8439 if (!VT.
isVector() && VT != MVT::x86mmx &&
8443 if (TLI.isTypeLegal(NewVT)) {
8445 return DAG.getExtractVectorElt(dl, VT, BitOp, 0);
8457 ElementCount NewNumElts =
8459 .divideCoefficientBy(EltSize);
8461 if (TLI.isTypeLegal(NewVT)) {
8463 return DAG.getExtractSubvector(dl, VT, BitOp, 0);
8468 return CreateStackStoreLoad(InOp, VT);
8476 SDValue WidenedOp = GetWidenedVector(
N->getOperand(1));
8477 return DAG.getNode(
ISD::FAKE_USE, SDLoc(), MVT::Other,
N->getOperand(0),
8482 EVT VT =
N->getValueType(0);
8484 EVT InVT =
N->getOperand(0).getValueType();
8489 unsigned NumOperands =
N->getNumOperands();
8490 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
8492 for (i = 1; i < NumOperands; ++i)
8493 if (!
N->getOperand(i).isUndef())
8496 if (i == NumOperands)
8497 return GetWidenedVector(
N->getOperand(0));
8503 for (
unsigned i = 0; i < NumOperands; ++i) {
8504 SDValue InOp = GetWidenedVector(
N->getOperand(i));
8506 InOp = DAG.getExtractSubvector(dl, InVT, InOp, 0);
8507 Result = DAG.getInsertSubvector(dl, Result, InOp, i * NumInElts);
8519 for (
unsigned i=0; i < NumOperands; ++i) {
8523 "Unexpected type action");
8524 InOp = GetWidenedVector(InOp);
8525 for (
unsigned j = 0;
j < NumInElts; ++
j)
8526 Ops[Idx++] = DAG.getExtractVectorElt(dl, EltVT, InOp, j);
8528 return DAG.getBuildVector(VT, dl,
Ops);
8533 EVT VT =
N->getValueType(0);
8535 EVT SrcVT = Src.getValueType();
8536 EVT WidenedSrcVT = TLI.getTypeToTransformTo(*DAG.getContext(), SrcVT);
8541 "Cannot widen VECTOR_REPEAT operand to an ElementCount that's not "
8542 "a known scalar multiple of the input ElementCount.");
8546 unsigned NumConcat =
8554 return DAG.getExtractSubvector(
DL, VT, Widened, 0);
8557SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(
SDNode *
N) {
8558 EVT VT =
N->getValueType(0);
8563 SubVec = GetWidenedVector(SubVec);
8568 bool IndicesValid =
false;
8571 IndicesValid =
true;
8575 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8576 Attribute::VScaleRange);
8581 IndicesValid =
true;
8587 "Don't know how to widen the operands for INSERT_SUBVECTOR");
8593 if (InVec.
isUndef() &&
N->getConstantOperandVal(2) == 0)
8600 if (SubVT == VT &&
N->getConstantOperandVal(2) == 0) {
8622 DAG.getStore(DAG.getEntryNode(),
DL, InVec, StackPtr, StoreMMO);
8630 TLI.getVectorSubVecPointer(DAG, StackPtr, VT, OrigVT,
N->getOperand(2));
8631 Ch = DAG.getMaskedStore(Ch,
DL, SubVec, SubVecPtr,
8636 return DAG.getLoad(VT,
DL, Ch, StackPtr, LoadMMO);
8641 unsigned Idx =
N->getConstantOperandVal(2);
8647 InsertElt = DAG.getInsertVectorElt(
DL, InsertElt, ExtractElt,
I + Idx);
8653SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
8654 SDValue InOp = GetWidenedVector(
N->getOperand(0));
8656 N->getValueType(0), InOp,
N->getOperand(1));
8659SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
8660 SDValue InOp = GetWidenedVector(
N->getOperand(0));
8662 N->getValueType(0), InOp,
N->getOperand(1));
8665SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(
SDNode *
N) {
8667 EVT ResVT =
N->getValueType(0);
8670 SDValue WideInOp = GetWidenedVector(
N->getOperand(0));
8676 return DAG.getNode(
N->getOpcode(),
DL, ResVT, WideInOp);
8684 "Widened input size must be a multiple of result element size");
8687 EVT WideResVT =
EVT::getVectorVT(*DAG.getContext(), ResEltVT, WideNumElts);
8689 SDValue WideRes = DAG.getNode(
N->getOpcode(),
DL, WideResVT, WideInOp);
8690 return DAG.getExtractSubvector(
DL, ResVT, WideRes, 0);
8698 if (!
ST->getMemoryVT().getScalarType().isByteSized())
8699 return TLI.scalarizeVectorStore(ST, DAG);
8701 if (
ST->isTruncatingStore())
8702 return TLI.scalarizeVectorStore(ST, DAG);
8712 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StVT);
8713 EVT WideMaskVT = getSetCCResultType(WideVT);
8715 if (TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) &&
8716 TLI.isTypeLegal(WideMaskVT)) {
8719 StVal = GetWidenedVector(StVal);
8721 SDValue EVL = DAG.getElementCount(
DL, TLI.getVPExplicitVectorLengthTy(),
8723 return DAG.getStoreVP(
ST->getChain(),
DL, StVal,
ST->getBasePtr(),
8724 ST->getOffset(), Mask, EVL, StVT,
ST->getMemOperand(),
8725 ST->getAddressingMode());
8729 if (GenWidenVectorStores(StChain, ST)) {
8730 if (StChain.
size() == 1)
8739 SDValue WideStVal = GetWidenedVector(StVal);
8743 return DAG.getMaskedStore(
ST->getChain(),
DL, WideStVal,
ST->getBasePtr(),
8744 ST->getOffset(), Mask,
ST->getMemoryVT(),
8745 ST->getMemOperand(),
ST->getAddressingMode(),
8746 ST->isTruncatingStore());
8753 EVT StVT =
ST->getMemoryVT();
8756 SDValue StVal = GetWidenedVector(
ST->getVal());
8761 TypeSize WidthDiff = WidenWidth - StWidth;
8767 std::optional<EVT> FirstVT =
8768 findMemType(DAG, TLI, StWidth.getKnownMinValue(), WidenVT, 0,
8773 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8779 ST->getBasePtr(),
ST->getMemOperand());
8782SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(
SDNode *
N,
unsigned OpNo) {
8783 assert((OpNo == 1 || OpNo == 3) &&
8784 "Can widen only data or mask operand of vp_store");
8792 StVal = GetWidenedVector(StVal);
8798 "Unable to widen VP store");
8799 Mask = GetWidenedVector(Mask);
8801 Mask = GetWidenedVector(Mask);
8807 "Unable to widen VP store");
8808 StVal = GetWidenedVector(StVal);
8811 assert(
Mask.getValueType().getVectorElementCount() ==
8813 "Mask and data vectors should have the same number of elements");
8814 return DAG.getStoreVP(
ST->getChain(), dl, StVal,
ST->getBasePtr(),
8815 ST->getOffset(), Mask,
ST->getVectorLength(),
8816 ST->getMemoryVT(),
ST->getMemOperand(),
8817 ST->getAddressingMode(),
ST->isTruncatingStore(),
8818 ST->isCompressingStore());
8823 assert((OpNo == 1 || OpNo == 4) &&
8824 "Can widen only data or mask operand of vp_strided_store");
8833 "Unable to widen VP strided store");
8837 "Unable to widen VP strided store");
8839 StVal = GetWidenedVector(StVal);
8840 Mask = GetWidenedVector(Mask);
8843 Mask.getValueType().getVectorElementCount() &&
8844 "Data and mask vectors should have the same number of elements");
8846 return DAG.getStridedStoreVP(
8853SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(
SDNode *
N,
unsigned OpNo) {
8854 assert((OpNo == 1 || OpNo == 4) &&
8855 "Can widen only data or mask operand of mstore");
8858 EVT MaskVT =
Mask.getValueType();
8863 EVT WideVT, WideMaskVT;
8866 StVal = GetWidenedVector(StVal);
8873 WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
8880 if (TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) &&
8882 Mask = DAG.getInsertSubvector(dl, DAG.getPOISON(WideMaskVT), Mask, 0);
8883 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
8892 Mask = ModifyToType(Mask, WideMaskVT,
true);
8895 Mask = ModifyToType(Mask, WideMaskVT,
true);
8897 StVal = ModifyToType(StVal, WideVT);
8900 assert(
Mask.getValueType().getVectorElementCount() ==
8902 "Mask and data vectors should have the same number of elements");
8909SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(
SDNode *
N,
unsigned OpNo) {
8910 assert(OpNo == 4 &&
"Can widen only the index of mgather");
8912 SDValue DataOp = MG->getPassThru();
8914 SDValue Scale = MG->getScale();
8922 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl,
Ops,
8923 MG->getMemOperand(), MG->getIndexType(),
8924 MG->getExtensionType());
8930SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(
SDNode *
N,
unsigned OpNo) {
8939 DataOp = GetWidenedVector(DataOp);
8943 EVT IndexVT =
Index.getValueType();
8946 Index = ModifyToType(Index, WideIndexVT);
8949 EVT MaskVT =
Mask.getValueType();
8952 Mask = ModifyToType(Mask, WideMaskVT,
true);
8957 }
else if (OpNo == 4) {
8959 Index = GetWidenedVector(Index);
8965 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(
N),
8970SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(
SDNode *
N,
unsigned OpNo) {
8979 DataOp = GetWidenedVector(DataOp);
8980 Index = GetWidenedVector(Index);
8982 Mask = GetWidenedMask(Mask, WideEC);
8985 }
else if (OpNo == 3) {
8987 Index = GetWidenedVector(Index);
8994 return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(
N),
Ops,
8999 SDValue InOp0 = GetWidenedVector(
N->getOperand(0));
9000 SDValue InOp1 = GetWidenedVector(
N->getOperand(1));
9002 EVT VT =
N->getValueType(0);
9017 SVT, InOp0, InOp1,
N->getOperand(2));
9023 SDValue CC = DAG.getExtractSubvector(dl, ResVT, WideSETCC, 0);
9025 EVT OpVT =
N->getOperand(0).getValueType();
9028 return DAG.getNode(ExtendCode, dl, VT,
CC);
9038 EVT VT =
N->getValueType(0);
9040 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
9047 for (
unsigned i = 0; i != NumElts; ++i) {
9048 SDValue LHSElem = DAG.getExtractVectorElt(dl, TmpEltVT,
LHS, i);
9049 SDValue RHSElem = DAG.getExtractVectorElt(dl, TmpEltVT,
RHS, i);
9051 Scalars[i] = DAG.getNode(
N->getOpcode(), dl, {MVT::i1, MVT::Other},
9052 {Chain, LHSElem, RHSElem, CC});
9053 Chains[i] = Scalars[i].getValue(1);
9054 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
9055 DAG.getBoolConstant(
true, dl, EltVT, VT),
9056 DAG.getBoolConstant(
false, dl, EltVT, VT));
9060 ReplaceValueWith(
SDValue(
N, 1), NewChain);
9062 return DAG.getBuildVector(VT, dl, Scalars);
9086 SDValue Op = GetWidenedVector(
N->getOperand(0));
9087 EVT VT =
N->getValueType(0);
9088 EVT OrigVT =
N->getOperand(0).getValueType();
9089 EVT WideVT =
Op.getValueType();
9091 SDNodeFlags
Flags =
N->getFlags();
9093 unsigned Opc =
N->getOpcode();
9095 SDValue NeutralElem = DAG.getIdentityElement(BaseOpc, dl, ElemVT, Flags);
9096 assert(NeutralElem &&
"Neutral element must exist");
9106 VPOpcode && TLI.isOperationLegalOrCustom(*VPOpcode, WideVT)) {
9113 SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
9114 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
9120 unsigned GCD = std::gcd(OrigElts, WideElts);
9123 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
9124 for (
unsigned Idx = OrigElts;
Idx < WideElts;
Idx =
Idx + GCD)
9125 Op = DAG.getInsertSubvector(dl,
Op, SplatNeutral, Idx);
9126 return DAG.getNode(
Opc, dl, VT,
Op, Flags);
9129 for (
unsigned Idx = OrigElts;
Idx < WideElts;
Idx++)
9130 Op = DAG.getInsertVectorElt(dl,
Op, NeutralElem, Idx);
9132 return DAG.getNode(
Opc, dl, VT,
Op, Flags);
9141 EVT VT =
N->getValueType(0);
9143 EVT WideVT =
Op.getValueType();
9145 SDNodeFlags
Flags =
N->getFlags();
9147 unsigned Opc =
N->getOpcode();
9149 SDValue NeutralElem = DAG.getIdentityElement(BaseOpc, dl, ElemVT, Flags);
9159 VPOpcode && TLI.isOperationLegalOrCustom(*VPOpcode, WideVT)) {
9162 SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
9163 SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
9169 unsigned GCD = std::gcd(OrigElts, WideElts);
9172 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
9173 for (
unsigned Idx = OrigElts;
Idx < WideElts;
Idx =
Idx + GCD)
9174 Op = DAG.getInsertSubvector(dl,
Op, SplatNeutral, Idx);
9175 return DAG.getNode(
Opc, dl, VT, AccOp,
Op, Flags);
9178 for (
unsigned Idx = OrigElts;
Idx < WideElts;
Idx++)
9179 Op = DAG.getInsertVectorElt(dl,
Op, NeutralElem, Idx);
9181 return DAG.getNode(
Opc, dl, VT, AccOp,
Op, Flags);
9185 assert(
N->isVPOpcode() &&
"Expected VP opcode");
9188 SDValue Op = GetWidenedVector(
N->getOperand(1));
9190 Op.getValueType().getVectorElementCount());
9192 return DAG.getNode(
N->getOpcode(), dl,
N->getValueType(0),
9193 {N->getOperand(0), Op, Mask, N->getOperand(3)},
9201 EVT VT =
N->getValueType(0);
9205 SDValue LeftIn = DAG.WidenVector(
N->getOperand(1), SDLoc(
N));
9206 SDValue RightIn = DAG.WidenVector(
N->getOperand(2), SDLoc(
N));
9211 return DAG.getExtractSubvector(
DL, VT,
Select, 0);
9217 EVT SourceVT =
Source.getValueType();
9218 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT);
9222 WideSource = GetWidenedVector(Source);
9229 WideSource = GetWidenedVector(Source);
9231 SmallVector<int>
Mask(WideElts);
9232 std::iota(
Mask.begin(),
Mask.end(), 0);
9234 Mask[
I] += WideElts;
9235 WideSource = DAG.getVectorShuffle(WideVT,
DL, WideSource,
AllOnes, Mask);
9237 WideSource = DAG.getInsertSubvector(
DL,
AllOnes, Source, 0);
9241 return DAG.
getNode(
N->getOpcode(),
DL,
N->getValueType(0), WideSource,
9248 EVT SrcVT =
Source.getValueType();
9252 return DAG.getNode(
N->getOpcode(),
DL,
N->getValueType(0),
9253 {Source, Mask, N->getOperand(2)},
N->getFlags());
9256SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(
SDNode *
N) {
9259 EVT OrigMaskVT =
Mask.getValueType();
9260 SDValue WideMask = GetWidenedVector(Mask);
9266 if (OrigElts != WideElts) {
9267 SDValue ZeroMask = DAG.getConstant(0,
DL, WideMaskVT);
9269 Mask, DAG.getVectorIdxConstant(0,
DL));
9276SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_SHUFFLE_VAR(
SDNode *
N,
9280 assert(OpNo == 1 &&
"Illegal operand must be mask");
9282 return TLI.expandVECTOR_SHUFFLE_VAR(
N, DAG);
9285SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(
SDNode *
N,
unsigned OpNo) {
9288 EVT ResVT =
N->getValueType(0);
9289 EVT SourceVT =
N->getOperand(0).getValueType();
9290 EVT WideSourceVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT);
9295 SDValue WideSource = DAG.getInsertSubvector(
DL, DAG.getUNDEF(WideSourceVT),
9296 N->getOperand(0), 0);
9297 SDValue WideMask = DAG.getInsertSubvector(
9298 DL, DAG.getConstant(0,
DL, WidenVT),
N->getOperand(2), 0);
9300 N->getOperand(1), WideMask,
N->getFlags());
9301 return DAG.getExtractSubvector(
DL, ResVT, WideMatch, 0);
9305 assert(OpNo == 1 &&
"Unexpected VECTOR_MATCH operand");
9311 return TLI.expandVectorMatch(
N, DAG);
9313 EVT WidenNeedleVT = TLI.getTypeToTransformTo(*DAG.getContext(), NeedleVT);
9317 SDValue WideNeedle = DAG.getSplatVector(WidenNeedleVT,
DL, Fill);
9318 WideNeedle = DAG.getInsertSubvector(
DL, WideNeedle, Needle, 0);
9321 N->getOperand(0), WideNeedle,
N->getOperand(2),
9325SDValue DAGTypeLegalizer::WidenVecOp_PARTIAL_REDUCE_MLA(
SDNode *
N) {
9328 return TLI.expandPartialReduceMLA(
N, DAG);
9345 unsigned WidenEx = 0) {
9350 unsigned AlignInBits =
Align*8;
9352 EVT RetVT = WidenEltVT;
9357 if (Width == WidenEltWidth)
9368 (WidenWidth % MemVTWidth) == 0 &&
9370 (MemVTWidth <= Width ||
9371 (
Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9372 if (MemVTWidth == WidenWidth)
9391 (WidenWidth % MemVTWidth) == 0 &&
9393 (MemVTWidth <= Width ||
9394 (
Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9403 return std::nullopt;
9414 unsigned Start,
unsigned End) {
9415 SDLoc dl(LdOps[Start]);
9416 EVT LdTy = LdOps[Start].getValueType();
9424 for (
unsigned i = Start + 1; i !=
End; ++i) {
9425 EVT NewLdTy = LdOps[i].getValueType();
9426 if (NewLdTy != LdTy) {
9445 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
LD->getValueType(0));
9446 EVT LdVT =
LD->getMemoryVT();
9456 MMOMetadata
Metadata =
LD->getMMOMetadataForSubAccess();
9460 TypeSize WidthDiff = WidenWidth - LdWidth;
9467 std::optional<EVT> FirstVT =
9468 findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT, LdAlign,
9475 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
9480 std::optional<EVT> NewVT = FirstVT;
9481 TypeSize RemainingWidth = LdWidth;
9482 TypeSize NewVTWidth = FirstVTWidth;
9484 RemainingWidth -= NewVTWidth;
9491 NewVTWidth = NewVT->getSizeInBits();
9497 SDValue LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr,
LD->getPointerInfo(),
9511 MachinePointerInfo MPI =
LD->getPointerInfo();
9517 for (EVT MemVT : MemVTs) {
9518 Align NewAlign = ScaledOffset == 0
9519 ?
LD->getBaseAlign()
9521 SDValue L = DAG.getLoad(MemVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags,
9541 EVT LdTy = LdOps[i].getValueType();
9544 for (--i; i >= 0; --i) {
9545 LdTy = LdOps[i].getValueType();
9552 ConcatOps[--
Idx] = LdOps[i];
9553 for (--i; i >= 0; --i) {
9554 EVT NewLdTy = LdOps[i].getValueType();
9555 if (NewLdTy != LdTy) {
9566 WidenOps[j] = ConcatOps[Idx+j];
9568 WidenOps[j] = DAG.getPOISON(LdTy);
9575 ConcatOps[--
Idx] = LdOps[i];
9586 SDValue UndefVal = DAG.getPOISON(LdTy);
9589 for (; i !=
End-
Idx; ++i)
9590 WidenOps[i] = ConcatOps[Idx+i];
9592 WidenOps[i] = UndefVal;
9603 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),
LD->getValueType(0));
9604 EVT LdVT =
LD->getMemoryVT();
9613 MMOMetadata
Metadata =
LD->getMMOMetadataForSubAccess();
9627 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr,
LD->getPointerInfo(),
9628 LdEltVT,
LD->getBaseAlign(), MMOFlags,
Metadata);
9634 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
9635 LD->getPointerInfo().getWithOffset(
Offset), LdEltVT,
9641 SDValue UndefVal = DAG.getPOISON(EltVT);
9642 for (; i != WidenNumElts; ++i)
9645 return DAG.getBuildVector(WidenVT, dl,
Ops);
9656 MMOMetadata
Metadata =
ST->getMMOMetadataForSubAccess();
9657 SDValue ValOp = GetWidenedVector(
ST->getValue());
9660 EVT StVT =
ST->getMemoryVT();
9668 "Mismatch between store and value types");
9672 MachinePointerInfo MPI =
ST->getPointerInfo();
9682 std::optional<EVT> NewVT =
9687 TypeSize NewVTWidth = NewVT->getSizeInBits();
9690 StWidth -= NewVTWidth;
9691 MemVTs.
back().second++;
9695 for (
const auto &Pair : MemVTs) {
9696 EVT NewVT = Pair.first;
9697 unsigned Count = Pair.second;
9703 Align NewAlign = ScaledOffset == 0
9704 ?
ST->getBaseAlign()
9706 SDValue EOp = DAG.getExtractSubvector(dl, NewVT, ValOp, Idx);
9707 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign,
9723 SDValue EOp = DAG.getExtractVectorElt(dl, NewVT, VecOp, Idx++);
9725 DAG.getStore(Chain, dl, EOp, BasePtr, MPI,
ST->getBaseAlign(),
9743 bool FillWithZeroes) {
9748 "input and widen element type must match");
9750 "cannot modify scalable vectors in this way");
9763 FillWithZeroes ? DAG.getConstant(0, dl, InVT) : DAG.getPOISON(InVT);
9765 for (
unsigned i = 1; i != NumConcat; ++i)
9772 return DAG.getExtractSubvector(dl, NVT, InOp, 0);
9778 unsigned CommonFactor = std::gcd(InNumElts, NewNumElts);
9783 unsigned NumCopiedParts = std::min(InNumElts, NewNumElts) / CommonFactor;
9784 for (
unsigned I = 0;
I != NumCopiedParts; ++
I)
9786 DAG.getExtractSubvector(dl, PartVT, InOp,
I * CommonFactor));
9788 unsigned NumResultParts = NewNumElts / CommonFactor;
9789 if (NumResultParts > NumCopiedParts) {
9790 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, PartVT)
9791 : DAG.getPOISON(PartVT);
9792 Ops.append(NumResultParts - NumCopiedParts, FillVal);
9799 "Scalable vectors should have been handled already.");
9807 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
9809 for (Idx = 0;
Idx < MinNumElts; ++
Idx)
9810 Ops[Idx] = DAG.getExtractVectorElt(dl, EltVT, InOp, Idx);
9812 SDValue UndefVal = DAG.getPOISON(EltVT);
9813 for (;
Idx < WidenNumElts; ++
Idx)
9814 Ops[Idx] = UndefVal;
9816 SDValue Widened = DAG.getBuildVector(NVT, dl,
Ops);
9817 if (!FillWithZeroes)
9821 "We expect to never want to FillWithZeroes for non-integral types.");
9824 MaskOps.
append(MinNumElts, DAG.getAllOnesConstant(dl, EltVT));
9825 MaskOps.
append(WidenNumElts - MinNumElts, DAG.getConstant(0, dl, EltVT));
9827 return DAG.getNode(
ISD::AND, dl, NVT, Widened,
9828 DAG.getBuildVector(NVT, dl, MaskOps));
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static constexpr Value * getValue(Ty &ValueOrUse)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static unsigned getExtendForIntVecReduction(SDNode *N)
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
static std::optional< EVT > findMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align, unsigned WidenEx)
static EVT getSETCCOperandType(SDValue N)
static bool isSETCCOp(unsigned Opcode)
static bool isLogicalMaskOp(unsigned Opcode)
static bool isSETCCorConvertedSETCC(SDValue N)
static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT, TypeSize FirstVTWidth, const SDLoc &dl, SelectionDAG &DAG)
Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the widened value so it can b...
static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, SmallVectorImpl< SDValue > &ConcatOps, unsigned ConcatEnd, EVT VT, EVT MaxVT, EVT WidenVT)
static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT, TypeSize LdWidth, TypeSize FirstVTWidth, SDLoc dl, SelectionDAG &DAG)
Either return the same load or provide appropriate casts from the load and return that.
static bool isUndef(const MachineInstr &MI)
This file provides utility analysis objects describing memory locations.
SmallVector< MachineOperand, 4 > Cond
const MachineOperand & HiOp1
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
This file implements the SmallBitVector class.
This is an SDNode representing atomic operations.
LLVM_ABI unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
bool isValid() const
Return true if the attribute is any kind of attribute.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
This class is used to represent ISD::LOAD nodes.
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
static auto integer_valuetypes()
static auto vector_valuetypes()
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Flags
Flags values. These may be or'd together.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
This class is used to represent an MGATHER node.
const SDValue & getIndex() const
const SDValue & getScale() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getInc() const
const SDValue & getScale() const
const SDValue & getMask() const
const SDValue & getIntID() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
ISD::MemIndexType getIndexType() const
This class is used to represent an MLOAD node.
const SDValue & getBasePtr() const
bool isExpandingLoad() const
ISD::LoadExtType getExtensionType() const
const SDValue & getMask() const
const SDValue & getPassThru() const
const SDValue & getOffset() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getOffset() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
This is an abstract virtual class for memory operations.
Align getBaseAlign() const
Returns alignment and volatility of the memory access.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
const MDNode * getMemCacheHint() const
Returns the cache hint metadata for this memory access.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDNodeFlags getFlags() const
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
SDValue getInsertVectorElt(const SDLoc &DL, SDValue Vec, SDValue Elt, unsigned Idx)
Insert Elt into Vec at offset Idx.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVMContext * getContext() const
size_type size() const
Determine the number of elements in the SetVector.
Vector takeVector()
Clear the SetVector and return the underlying vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
@ TypeScalarizeScalableVector
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
This class is used to represent an VP_GATHER node.
const SDValue & getScale() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getVectorLength() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
This class is used to represent a VP_LOAD node.
const SDValue & getValue() const
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
const SDValue & getMask() const
ISD::LoadExtType getExtensionType() const
bool isExpandingLoad() const
const SDValue & getStride() const
const SDValue & getOffset() const
const SDValue & getVectorLength() const
const SDValue & getBasePtr() const
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if this is a truncating store.
const SDValue & getOffset() const
const SDValue & getVectorLength() const
const SDValue & getStride() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isNonZero() const
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS*X will result in a value whose quantity matches our own.
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ ATOMIC_LOAD_FMINIMUMNUM
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ AVGSUBFLOORS
AVGSUBFLOORS/AVGSUBFLOORU - Averaging subtract - Subtract two integers using an integer of type i[N+1...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ CLMUL
Carry-less multiplication operations.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ ATOMIC_LOAD_FMAXIMUMNUM
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Experimental vector histogram intrinsic Operands: Input Chain, Inc, Mask, Base, Index,...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
LLVM_ABI NodeType getUnmaskedBinOpOpcode(unsigned MaskedOpc)
Given a MaskedOpc of ISD::MASKED_(U|S)(DIV|REM), returns the unmasked ISD::(U|S)(DIV|REM).
LLVM_ABI bool isBuildVectorAllOnes(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LLVM_ABI std::optional< unsigned > getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
LLVM_ABI LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
auto reverse(ContainerTy &&C)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr int PoisonMaskElem
@ Mul
Product of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
OutputIt copy(R &&Range, OutputIt Out)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLVM_ABI void processShuffleMasks(ArrayRef< int > Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, unsigned NumOfUsedRegs, function_ref< void()> NoInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> SingleInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned, bool)> ManyInputsAction)
Splits and processes shuffle mask depending on the number of input and output registers.
@ Increment
Incrementally increasing token ID.
@ End
Run the machine verifier once, at the end of the pipeline.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
EVT changeVectorElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element coun...
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
bool knownBitsGE(EVT VT) const
Return true if we know at compile time this has more than or the same bits as VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.