32#define DEBUG_TYPE "legalize-types"
42void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
47 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
52 switch (
N->getOpcode()) {
55 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
56 N->dump(&DAG);
dbgs() <<
"\n";
63 case ISD::VP_BITREVERSE:
66 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
69 case ISD::VP_CTLZ_ZERO_UNDEF:
72 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
75 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
76 case ISD::VP_CTTZ_ZERO_UNDEF:
79 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
80 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
81 case ISD::VP_CTTZ_ELTS:
82 Res = PromoteIntRes_VP_CttzElements(
N);
85 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
86 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
88 Res = PromoteIntRes_VP_LOAD(cast<VPLoadSDNode>(
N));
90 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
92 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
95 Res = PromoteIntRes_VECTOR_COMPRESS(
N);
101 Res = PromoteIntRes_Select(
N);
106 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
108 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
110 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
113 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
115 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
117 case ISD::VP_SRA: Res = PromoteIntRes_SRA(
N);
break;
119 case ISD::VP_SRL: Res = PromoteIntRes_SRL(
N);
break;
120 case ISD::VP_TRUNCATE:
123 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
124 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
128 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
130 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
132 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
134 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
136 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
139 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
142 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
144 Res = PromoteIntRes_BUILD_VECTOR(
N);
148 case ISD::EXPERIMENTAL_VP_SPLAT:
149 Res = PromoteIntRes_ScalarOp(
N);
153 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
158 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
161 Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(
N);
165 Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(
N);
171 Res = PromoteIntRes_PARTIAL_REDUCE_MLA(
N);
175 case ISD::VP_SIGN_EXTEND:
177 case ISD::VP_ZERO_EXTEND:
180 case ISD::VP_FP_TO_SINT:
181 case ISD::VP_FP_TO_UINT:
189 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
193 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
197 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
212 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
222 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
232 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
235 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
237 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
239 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
255 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
257 case ISD::VP_SADDSAT:
258 case ISD::VP_UADDSAT:
259 case ISD::VP_SSUBSAT:
260 case ISD::VP_USUBSAT:
261 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
266 Res = PromoteIntRes_CMP(
N);
279 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
282 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
296 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
300 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
312 Res = PromoteIntRes_VECREDUCE(
N);
315 case ISD::VP_REDUCE_ADD:
316 case ISD::VP_REDUCE_MUL:
317 case ISD::VP_REDUCE_AND:
318 case ISD::VP_REDUCE_OR:
319 case ISD::VP_REDUCE_XOR:
320 case ISD::VP_REDUCE_SMAX:
321 case ISD::VP_REDUCE_SMIN:
322 case ISD::VP_REDUCE_UMAX:
323 case ISD::VP_REDUCE_UMIN:
324 Res = PromoteIntRes_VP_REDUCE(
N);
329 Res = PromoteIntRes_LOOP_DEPENDENCE_MASK(
N);
333 Res = PromoteIntRes_FREEZE(
N);
338 Res = PromoteIntRes_Rotate(
N);
343 Res = PromoteIntRes_FunnelShift(
N);
348 Res = PromoteIntRes_VPFunnelShift(
N);
352 Res = PromoteIntRes_IS_FPCLASS(
N);
355 Res = PromoteIntRes_FFREXP(
N);
360 Res = PromoteIntRes_XRINT(
N);
364 Res = PromoteIntRes_PATCHPOINT(
N);
367 Res = PromoteIntRes_READ_REGISTER(
N);
373 SetPromotedInteger(
SDValue(
N, ResNo), Res);
378 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
379 return GetPromotedInteger(
Op);
382SDValue DAGTypeLegalizer::PromoteIntRes_LOOP_DEPENDENCE_MASK(
SDNode *
N) {
383 EVT VT =
N->getValueType(0);
390 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
392 Op.getValueType(),
Op,
N->getOperand(1));
397 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
399 Op.getValueType(),
Op,
N->getOperand(1));
423 N->getChain(),
N->getBasePtr(),
N->getMemOperand());
432 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
435 N->getChain(),
N->getBasePtr(),
436 Op2,
N->getMemOperand());
447 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
458 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
468 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
471 Op2 = SExtPromotedInteger(Op2);
474 Op2 = ZExtPromotedInteger(Op2);
477 Op2 = GetPromotedInteger(Op2);
486 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
487 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
489 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
498 EVT OutVT =
N->getValueType(0);
502 switch (getTypeAction(InVT)) {
529 BitConvertToInteger(GetScalarizedVector(InOp)));
538 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
539 Lo = BitConvertToInteger(
Lo);
540 Hi = BitConvertToInteger(
Hi);
548 JoinIntegers(
Lo,
Hi));
582 if (isTypeLegal(WideOutVT)) {
583 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
605 if (isTypeLegal(WideVecVT)) {
616 CreateStackStoreLoad(InOp, OutVT));
620 SDValue V = GetPromotedInteger(
N->getOperand(0));
622 V.getValueType(), V);
626 SDValue Op = GetPromotedInteger(
N->getOperand(0));
627 EVT OVT =
N->getValueType(0);
628 EVT NVT =
Op.getValueType();
648 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
649 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
654 SDValue Op = GetPromotedInteger(
N->getOperand(0));
655 EVT OVT =
N->getValueType(0);
656 EVT NVT =
Op.getValueType();
676 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
677 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
686 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
691 EVT VT =
N->getValueType(0);
700 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
705 EVT OVT =
N->getValueType(0);
721 unsigned CtlzOpcode =
N->getOpcode();
722 if (CtlzOpcode ==
ISD::CTLZ || CtlzOpcode == ISD::VP_CTLZ) {
727 if (!
N->isVPOpcode()) {
729 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
737 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
738 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
739 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
740 ExtractLeadingBits, Mask, EVL);
743 CtlzOpcode == ISD::VP_CTLZ_ZERO_UNDEF) {
745 SDValue Op = GetPromotedInteger(
N->getOperand(0));
750 if (!
N->isVPOpcode()) {
752 return DAG.
getNode(CtlzOpcode, dl, NVT,
Op);
757 Op = DAG.
getNode(ISD::VP_SHL, dl, NVT,
Op, ShiftConst, Mask, EVL);
758 return DAG.
getNode(CtlzOpcode, dl, NVT,
Op, Mask, EVL);
764 EVT OVT =
N->getValueType(0);
781 if (!
N->isVPOpcode()) {
782 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
788 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
794 SDValue Op = GetPromotedInteger(
N->getOperand(0));
795 EVT OVT =
N->getValueType(0);
796 EVT NVT =
Op.getValueType();
814 unsigned NewOpc =
N->getOpcode();
815 if (NewOpc ==
ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
827 N->getOperand(1),
N->getOperand(2));
828 NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
831 if (!
N->isVPOpcode())
833 return DAG.
getNode(NewOpc, dl, NVT,
Op,
N->getOperand(1),
N->getOperand(2));
836SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(
SDNode *
N) {
839 return DAG.
getNode(
N->getOpcode(),
DL, NewVT,
N->ops());
842SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
857 EVT SVT =
In.getValueType().getScalarType();
874 if (
N->isStrictFPOpcode()) {
875 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
876 {
N->getOperand(0),
N->getOperand(1)});
880 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
881 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
884 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
896 N->getOpcode() == ISD::VP_FP_TO_UINT)
903SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
907 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
911SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
915 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
918SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
923 N->getOperand(0),
N->getOperand(1));
931 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
939 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
951 if (getTypeAction(
N->getOperand(0).getValueType())
953 SDValue Res = GetPromotedInteger(
N->getOperand(0));
971 if (
N->getNumOperands() != 1) {
972 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
973 assert(
N->isVPOpcode() &&
"Expected VP opcode");
974 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
975 N->getOperand(1),
N->getOperand(2));
977 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
987 N->getMemoryVT(),
N->getMemOperand());
996 assert(!
N->isIndexed() &&
"Indexed vp_load during type legalization!");
1000 :
N->getExtensionType();
1003 DAG.
getExtLoadVP(ExtType, dl, NVT,
N->getChain(),
N->getBasePtr(),
1004 N->getMask(),
N->getVectorLength(),
N->getMemoryVT(),
1005 N->getMemOperand(),
N->isExpandingLoad());
1014 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
1022 N->getOffset(),
N->getMask(), ExtPassThru,
1023 N->getMemoryVT(),
N->getMemOperand(),
1024 N->getAddressingMode(), ExtType,
1025 N->isExpandingLoad());
1034 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
1036 "Gather result type and the passThru argument type should be the same");
1043 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
1044 N->getIndex(),
N->getScale() };
1046 N->getMemoryVT(), dl, Ops,
1047 N->getMemOperand(),
N->getIndexType(),
1055SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(
SDNode *
N) {
1056 SDValue Vec = GetPromotedInteger(
N->getOperand(0));
1057 SDValue Passthru = GetPromotedInteger(
N->getOperand(2));
1059 N->getOperand(1), Passthru);
1067 EVT VT =
N->getValueType(0);
1068 EVT SVT = getSetCCResultType(VT);
1069 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
1070 unsigned NumOps =
N->getNumOperands();
1071 assert(NumOps <= 3 &&
"Too many operands");
1073 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
1081 ReplaceValueWith(
SDValue(
N, 0), Res);
1087template <
class MatchContextClass>
1099 MatchContextClass matcher(DAG, TLI,
N);
1101 unsigned Opcode = matcher.getRootBaseOpcode();
1107 SExtOrZExtPromotedOperands(Op1, Op2);
1116 Op1 = SExtPromotedInteger(Op1);
1117 Op2 = SExtPromotedInteger(Op2);
1118 return matcher.getNode(
ISD::UADDSAT, dl, NVT, Op1, Op2);
1121 Op1 = ZExtPromotedInteger(Op1);
1122 Op2 = ZExtPromotedInteger(Op2);
1127 return matcher.getNode(
ISD::UMIN, dl, NVT,
Add, SatMax);
1134 Op1 = GetPromotedInteger(Op1);
1135 Op2 = ZExtPromotedInteger(Op2);
1137 Op1 = SExtPromotedInteger(Op1);
1138 Op2 = SExtPromotedInteger(Op2);
1145 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1158 "addition, subtraction or left shift");
1161 unsigned SHLAmount = NewBits - OldBits;
1168 SDValue Result = matcher.getNode(Opcode, dl, PromotedType, Op1, Op2);
1169 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1177 SDValue Result = matcher.getNode(AddOp, dl, PromotedType, Op1, Op2);
1178 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1179 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1186 SDValue Op1Promoted, Op2Promoted;
1192 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1193 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1195 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1196 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1198 EVT OldType =
N->getOperand(0).getValueType();
1212 Op2Promoted,
N->getOperand(2));
1214 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1217 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1222 unsigned SatW,
bool Signed,
1225 EVT VT = V.getValueType();
1252 EVT VT =
LHS.getValueType();
1270 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1276 "Tried to saturate to more than the original type?");
1285 SDValue Op1Promoted, Op2Promoted;
1291 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1292 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1294 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1295 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1298 unsigned Scale =
N->getConstantOperandVal(2);
1307 N->getValueType(0).getScalarSizeInBits();
1312 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1313 Op2Promoted,
N->getOperand(2));
1323 Op2Promoted, Scale, DAG)) {
1326 N->getValueType(0).getScalarSizeInBits(),
1334 N->getValueType(0).getScalarSizeInBits());
1337SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1339 return PromoteIntRes_Overflow(
N);
1343 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1344 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1345 EVT OVT =
N->getOperand(0).getValueType();
1346 EVT NVT =
LHS.getValueType();
1361 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1367 EVT PromotedResultTy =
1370 N->getOperand(0),
N->getOperand(1));
1376 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1377 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1379 unsigned Opcode =
N->getOpcode();
1380 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1387 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1388 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1390 LHS.getValueType(),
N->getOperand(0),
1391 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1395 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1396 EVT InVT =
N->getOperand(OpNo).getValueType();
1399 EVT SVT = getSetCCResultType(InVT);
1407 SVT = getSetCCResultType(InVT);
1415 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1416 "Vector compare must return a vector result!");
1420 if (
N->isStrictFPOpcode()) {
1422 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1423 N->getOperand(2),
N->getOperand(3)};
1424 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1429 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1430 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1446 EVT VT =
N->getValueType(0);
1452 ReplaceValueWith(
SDValue(
N, 0), Res);
1457 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1459 if (
N->getOpcode() != ISD::VP_SHL) {
1461 RHS = ZExtPromotedInteger(RHS);
1469 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1474SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1475 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1477 Op.getValueType(),
Op,
N->getOperand(1));
1480SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1484 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1485 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1486 if (
N->getNumOperands() == 2)
1488 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1489 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1491 N->getOperand(2),
N->getOperand(3));
1495 if (
N->getNumOperands() == 2) {
1497 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1498 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1501 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1502 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1506 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1507 SDValue RHS = VPSExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1513 if (
N->getNumOperands() == 2) {
1515 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1516 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1519 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1520 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1524 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1525 SDValue RHS = VPZExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1536 SExtOrZExtPromotedOperands(LHS, RHS);
1539 LHS.getValueType(), LHS, RHS);
1544 if (
N->getOpcode() != ISD::VP_SRA) {
1546 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1548 RHS = ZExtPromotedInteger(RHS);
1555 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1557 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1564 if (
N->getOpcode() != ISD::VP_SRL) {
1566 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1568 RHS = ZExtPromotedInteger(RHS);
1575 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1577 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1585 ReplaceValueWith(
SDValue(
N, 0), Res);
1590 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1591 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1594 Amt = ZExtPromotedInteger(Amt);
1598 EVT OldVT =
N->getOperand(0).getValueType();
1599 EVT VT =
Lo.getValueType();
1600 unsigned Opcode =
N->getOpcode();
1614 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1639SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1640 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1641 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1646 Amt = VPZExtPromotedInteger(Amt, Mask, EVL);
1650 EVT OldVT =
N->getOperand(0).getValueType();
1651 EVT VT =
Lo.getValueType();
1652 unsigned Opcode =
N->getOpcode();
1653 bool IsFSHR = Opcode == ISD::VP_FSHR;
1658 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1666 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1669 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1672 Res = DAG.
getNode(IsFSHR ? ISD::VP_SRL : ISD::VP_SHL,
DL, VT, Res, Amt,
1675 Res = DAG.
getNode(ISD::VP_SRL,
DL, VT, Res, HiShift, Mask, EVL);
1681 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1686 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1704 Res = GetPromotedInteger(InOp);
1711 "Dst and Src must have the same number of elements");
1713 "Promoted vector type must be a power of two");
1716 GetSplitVector(InOp, EOp1, EOp2);
1724 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1725 "Expected VP_TRUNCATE opcode");
1726 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1727 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1728 std::tie(EVLLo, EVLHi) =
1729 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1730 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1731 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1738 SDValue WideInOp = GetWidenedVector(InOp);
1743 N->getValueType(0).getScalarType(), NumElem);
1758 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1759 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1764SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1766 return PromoteIntRes_Overflow(
N);
1770 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1771 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1772 EVT OVT =
N->getOperand(0).getValueType();
1773 EVT NVT =
LHS.getValueType();
1787 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1798 return PromoteIntRes_Overflow(
N);
1810 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1811 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1813 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1817 LHS, RHS,
N->getOperand(2));
1827 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1828 return PromoteIntRes_Overflow(
N);
1832 EVT OVT =
N->getValueType(0);
1845 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1849SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1852 return PromoteIntRes_Overflow(
N);
1856 EVT SmallVT =
LHS.getValueType();
1863 LHS = SExtPromotedInteger(LHS);
1864 RHS = SExtPromotedInteger(RHS);
1866 LHS = ZExtPromotedInteger(LHS);
1867 RHS = ZExtPromotedInteger(RHS);
1898 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1904 N->getValueType(0)));
1910 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1917 EVT VT =
N->getValueType(0);
1925 for (
unsigned i = 0; i < NumRegs; ++i) {
1926 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1927 N->getConstantOperandVal(3));
1933 std::reverse(Parts.begin(), Parts.end());
1938 for (
unsigned i = 1; i < NumRegs; ++i) {
1949 ReplaceValueWith(
SDValue(
N, 1), Chain);
1962bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1965 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1970 switch (
N->getOpcode()) {
1973 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1974 N->dump(&DAG);
dbgs() <<
"\n";
1980 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1983 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1984 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1990 Res = PromoteIntOp_FAKE_USE(
N);
1993 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1997 case ISD::EXPERIMENTAL_VP_SPLAT:
1998 Res = PromoteIntOp_ScalarOp(
N);
2001 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
2004 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
2006 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
2007 case ISD::VP_SINT_TO_FP:
2010 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
2013 Res = PromoteIntOp_VP_STORE(cast<VPStoreSDNode>(
N), OpNo);
2015 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
2017 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
2019 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
2021 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
2024 Res = PromoteIntOp_VECTOR_COMPRESS(
N, OpNo);
2026 case ISD::VP_TRUNCATE:
2030 case ISD::VP_UINT_TO_FP:
2035 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
2043 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
2046 case ISD::UCMP: Res = PromoteIntOp_CMP(
N);
break;
2049 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
2075 case ISD::VP_REDUCE_ADD:
2076 case ISD::VP_REDUCE_MUL:
2077 case ISD::VP_REDUCE_AND:
2078 case ISD::VP_REDUCE_OR:
2079 case ISD::VP_REDUCE_XOR:
2080 case ISD::VP_REDUCE_SMAX:
2081 case ISD::VP_REDUCE_SMIN:
2082 case ISD::VP_REDUCE_UMAX:
2083 case ISD::VP_REDUCE_UMIN:
2084 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
2089 Res = PromoteIntOp_STACKMAP(
N, OpNo);
2092 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
2095 Res = PromoteIntOp_WRITE_REGISTER(
N, OpNo);
2097 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2098 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2099 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
2101 case ISD::EXPERIMENTAL_VP_SPLICE:
2102 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
2105 Res = PromoteIntOp_VECTOR_HISTOGRAM(
N, OpNo);
2108 Res = PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(
N, OpNo);
2111 Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(
N);
2116 Res = PromoteIntOp_PARTIAL_REDUCE_MLA(
N);
2121 if (!Res.
getNode())
return false;
2128 const bool IsStrictFp =
N->isStrictFPOpcode();
2130 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2131 "Invalid operand expansion");
2135 ReplaceValueWith(
SDValue(
N, 0), Res);
2145void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
2146 SDValue OpL = GetPromotedInteger(LHS);
2147 SDValue OpR = GetPromotedInteger(RHS);
2153 unsigned OpLEffectiveBits =
2155 unsigned OpREffectiveBits =
2157 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2158 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2165 LHS = SExtPromotedInteger(LHS);
2166 RHS = SExtPromotedInteger(RHS);
2177 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2178 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2185 LHS = ZExtPromotedInteger(LHS);
2186 RHS = ZExtPromotedInteger(RHS);
2191void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2200 LHS = SExtPromotedInteger(LHS);
2201 RHS = SExtPromotedInteger(RHS);
2206 "Unknown integer comparison!");
2208 SExtOrZExtPromotedOperands(LHS, RHS);
2212 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2217 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2219 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2223 EVT OutVT =
N->getValueType(0);
2229 switch (getTypeAction(InVT)) {
2243 if (isTypeLegal(WideVecVT)) {
2244 SDValue Promoted = GetPromotedInteger(InOp);
2260 return CreateStackStoreLoad(InOp, OutVT);
2263SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2264 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2268 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2273 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2277SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2278 assert(OpNo == 1 &&
"only know how to promote condition");
2281 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2285 N->getOperand(2)), 0);
2290 EVT OVT =
N->getOperand(0).getValueType();
2291 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2292 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2293 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2306 EVT VecVT =
N->getValueType(0);
2309 "Legal vector of one illegal element?");
2314 assert(
N->getOperand(0).getValueSizeInBits() >=
2315 N->getValueType(0).getScalarSizeInBits() &&
2316 "Type of inserted value narrower than vector element type!");
2319 for (
unsigned i = 0; i < NumElts; ++i)
2320 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2325SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2332 assert(
N->getOperand(1).getValueSizeInBits() >=
2333 N->getValueType(0).getScalarSizeInBits() &&
2334 "Type of inserted value narrower than vector element type!");
2336 GetPromotedInteger(
N->getOperand(1)),
2341 assert(OpNo == 2 &&
"Different operand and result vector types?");
2347 N->getOperand(1),
Idx), 0);
2351 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2352 if (
N->getOpcode() == ISD::EXPERIMENTAL_VP_SPLAT)
2361SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2362 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2364 EVT OpTy =
N->getOperand(1).getValueType();
2367 if (
SDValue Res = WidenVSELECTMask(
N))
2369 Res,
N->getOperand(1),
N->getOperand(2));
2373 Cond = PromoteTargetBoolean(
Cond, OpVT);
2376 N->getOperand(2)), 0);
2379SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2380 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2384 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2388 N->getOperand(3),
N->getOperand(4)), 0);
2391SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2392 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2396 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2402 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2405 N->getOperand(3),
N->getOperand(4)),
2411 ZExtPromotedInteger(
N->getOperand(1))), 0);
2419 LHS = SExtPromotedInteger(LHS);
2420 RHS = SExtPromotedInteger(RHS);
2422 SExtOrZExtPromotedOperands(LHS, RHS);
2430 ZExtPromotedInteger(
N->getOperand(2))), 0);
2434 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2441SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2443 EVT VT =
N->getValueType(0);
2444 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2446 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2454 return DAG.
getNode(ISD::VP_SRA, dl, VT, Shl, ShAmt,
N->getOperand(1),
2459 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2461 SExtPromotedInteger(
N->getOperand(0)),
2462 N->getOperand(1),
N->getOperand(2)),
2465 SExtPromotedInteger(
N->getOperand(0))), 0);
2468SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2470 SExtPromotedInteger(
N->getOperand(1))), 0);
2478 SDValue Val = GetPromotedInteger(
N->getValue());
2482 N->getMemoryVT(),
N->getMemOperand());
2488 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2489 assert(!
N->isIndexed() &&
"expecting unindexed vp_store!");
2491 SDValue DataOp = GetPromotedInteger(
N->getValue());
2493 N->getMask(),
N->getVectorLength(),
2494 N->getMemoryVT(),
N->getMemOperand(),
2495 N->isCompressingStore());
2506 Mask = PromoteTargetBoolean(Mask, DataVT);
2512 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2513 DataOp = GetPromotedInteger(DataOp);
2516 N->getOffset(), Mask,
N->getMemoryVT(),
2517 N->getMemOperand(),
N->getAddressingMode(),
2518 true,
N->isCompressingStore());
2523 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2524 EVT DataVT =
N->getValueType(0);
2525 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2527 NewOps[OpNo] =
Mask;
2544 EVT DataVT =
N->getValueType(0);
2545 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2546 }
else if (OpNo == 4) {
2548 if (
N->isIndexSigned())
2550 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2552 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2554 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2568 bool TruncateStore =
N->isTruncatingStore();
2573 EVT DataVT =
N->getValue().getValueType();
2574 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2575 }
else if (OpNo == 4) {
2577 if (
N->isIndexSigned())
2579 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2581 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2583 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2584 TruncateStore =
true;
2588 SDLoc(
N), NewOps,
N->getMemOperand(),
2589 N->getIndexType(), TruncateStore);
2594 assert(OpNo == 1 &&
"Can only promote VECTOR_COMPRESS mask.");
2597 SDValue Passthru =
N->getOperand(2);
2598 SDValue Mask = PromoteTargetBoolean(
N->getOperand(1), VT);
2603 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2604 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2606 N->getOperand(1),
N->getOperand(2));
2611 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2613 ZExtPromotedInteger(
N->getOperand(0)),
2614 N->getOperand(1),
N->getOperand(2)),
2617 ZExtPromotedInteger(
N->getOperand(0))), 0);
2620SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2622 ZExtPromotedInteger(
N->getOperand(1))), 0);
2629 EVT VT =
N->getValueType(0);
2634 if (
N->getFlags().hasNonNeg() &&
Op.getValueType() == VT &&
2637 if (OpEffectiveBits <= Src.getScalarValueSizeInBits())
2645SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2647 EVT VT =
N->getValueType(0);
2648 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2650 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2653 N->getOperand(0).getValueType());
2657 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2662SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2664 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2669 bool IsStrict =
N->isStrictFPOpcode();
2674 unsigned OpOffset = IsStrict ? 1 : 0;
2687 if (IsPowI &&
N->getValueType(0).isVector())
2690 NewOps[1 + OpOffset] = SExtPromotedInteger(
N->getOperand(1 + OpOffset));
2702 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2703 "POWI exponent should match with sizeof(int) when doing the libcall.");
2706 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2707 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2708 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2709 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2711 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2716 switch (
N->getOpcode()) {
2724 case ISD::VP_REDUCE_ADD:
2725 case ISD::VP_REDUCE_MUL:
2726 case ISD::VP_REDUCE_AND:
2727 case ISD::VP_REDUCE_OR:
2728 case ISD::VP_REDUCE_XOR:
2732 case ISD::VP_REDUCE_SMAX:
2733 case ISD::VP_REDUCE_SMIN:
2737 case ISD::VP_REDUCE_UMAX:
2738 case ISD::VP_REDUCE_UMIN:
2748 return GetPromotedInteger(V);
2750 return SExtPromotedInteger(V);
2752 return ZExtPromotedInteger(V);
2758 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2760 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2761 EVT InVT =
Op.getValueType();
2763 EVT ResVT =
N->getValueType(0);
2764 unsigned Opcode =
N->getOpcode();
2784 Op = ZExtPromotedInteger(
N->getOperand(0));
2787 Op = SExtPromotedInteger(
N->getOperand(0));
2803 Op = ZExtPromotedInteger(
N->getOperand(0));
2806 Op = SExtPromotedInteger(
N->getOperand(0));
2820SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2827 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2831 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2833 Op = PromoteIntOpVectorReduction(
N,
Op);
2837 EVT VT =
N->getValueType(0);
2838 EVT EltVT =
Op.getValueType().getScalarType();
2853 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2857SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2860 SDValue Operand =
N->getOperand(OpNo);
2866SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2869 SDValue Operand =
N->getOperand(OpNo);
2879 "cannot use llvm.write_register with illegal type", Fn,
2881 return N->getOperand(0);
2884SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2885 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2886 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2889 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2894SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2898 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2902 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2904 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2908SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_HISTOGRAM(
SDNode *
N,
2910 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2912 NewOps[1] = GetPromotedInteger(
N->getOperand(1));
2916SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(
SDNode *
N,
2919 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2923SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(
SDNode *
N) {
2925 NewOps[0] = ZExtPromotedInteger(
N->getOperand(0));
2926 NewOps[1] = ZExtPromotedInteger(
N->getOperand(1));
2930SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_MLA(
SDNode *
N) {
2932 switch (
N->getOpcode()) {
2934 NewOps[1] = SExtPromotedInteger(
N->getOperand(1));
2935 NewOps[2] = SExtPromotedInteger(
N->getOperand(2));
2938 NewOps[1] = ZExtPromotedInteger(
N->getOperand(1));
2939 NewOps[2] = ZExtPromotedInteger(
N->getOperand(2));
2942 NewOps[1] = SExtPromotedInteger(
N->getOperand(1));
2943 NewOps[2] = ZExtPromotedInteger(
N->getOperand(2));
2959void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2965 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2968 switch (
N->getOpcode()) {
2971 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2972 N->dump(&DAG);
dbgs() <<
"\n";
3022 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
3049 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
3050 SplitInteger(Tmp.first,
Lo,
Hi);
3051 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3059 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3068 SplitInteger(Tmp,
Lo,
Hi);
3147 ExpandIntRes_Rotate(
N,
Lo,
Hi);
3152 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
3156 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
3160 ExpandIntRes_READ_REGISTER(
N,
Lo,
Hi);
3170std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
3171 unsigned Opc =
Node->getOpcode();
3177 EVT RetVT =
Node->getValueType(0);
3185 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3186 "Unexpected atomic op or value type!");
3190 Node->getOperand(0));
3195void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
3200 GetExpandedInteger(
N->getOperand(0), InL, InH);
3211 unsigned VTBits =
N->getValueType(0).getSizeInBits();
3215 if (Amt.
uge(VTBits)) {
3217 }
else if (Amt.
ugt(NVTBits)) {
3221 }
else if (Amt == NVTBits) {
3238 if (Amt.
uge(VTBits)) {
3240 }
else if (Amt.
ugt(NVTBits)) {
3244 }
else if (Amt == NVTBits) {
3261 if (Amt.
uge(VTBits)) {
3264 }
else if (Amt.
ugt(NVTBits)) {
3269 }
else if (Amt == NVTBits) {
3289bool DAGTypeLegalizer::
3291 unsigned Opc =
N->getOpcode();
3299 "Expanded integer type size not a power of two!");
3306 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
3311 GetExpandedInteger(In, InL, InH);
3378bool DAGTypeLegalizer::
3385 "Expanded integer type size not a power of two!");
3390 GetExpandedInteger(
N->getOperand(0), InL, InH);
3402 switch (
N->getOpcode()) {
3417 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3433 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3449 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3475 EVT NewVT = getSetCCResultType(
LHS.getValueType());
3481 SplitInteger(Res,
Lo,
Hi);
3484void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3493 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3494 unsigned NumHalfBits = NumBits / 2;
3497 SDValue LHSL, LHSH, RHSL, RHSH;
3498 GetExpandedInteger(LHS, LHSL, LHSH);
3499 GetExpandedInteger(RHS, RHSL, RHSH);
3512 SDValue LHSL, LHSH, RHSL, RHSH;
3513 GetExpandedInteger(LHS, LHSL, LHSH);
3514 GetExpandedInteger(RHS, RHSL, RHSH);
3516 EVT CCT = getSetCCResultType(NVT);
3525 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3529 const APInt *RHSVal =
nullptr;
3530 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3531 RHSVal = &RHSConst->getAPIntValue();
3538 SDValue LHSL, LHSH, RHSL, RHSH;
3539 GetExpandedInteger(LHS, LHSL, LHSH);
3540 GetExpandedInteger(RHS, RHSL, RHSH);
3542 EVT CCT = getSetCCResultType(NVT);
3548 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3567 switch (
N->getOpcode()) {
3594 EVT VT =
N->getValueType(0);
3595 EVT CCT = getSetCCResultType(VT);
3598 SplitInteger(Result,
Lo,
Hi);
3603 SplitInteger(ExpandedCMP,
Lo,
Hi);
3606void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3610 SDValue LHSL, LHSH, RHSL, RHSH;
3611 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3612 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3615 SDValue LoOps[2] = { LHSL, RHSL };
3616 SDValue HiOps[3] = { LHSH, RHSH };
3625 HiOps[2] =
Lo.getValue(1);
3631 HiOps[2] =
Lo.getValue(1);
3653 HiOps[2] =
Lo.getValue(1);
3657 HiOps[2] =
Lo.getValue(1);
3670 EVT OvfVT = getSetCCResultType(NVT);
3709 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3712 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3715 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3749void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3752 SDValue LHSL, LHSH, RHSL, RHSH;
3754 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3755 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3757 SDValue LoOps[2] = { LHSL, RHSL };
3758 SDValue HiOps[3] = { LHSH, RHSH };
3762 HiOps[2] =
Lo.getValue(1);
3766 HiOps[2] =
Lo.getValue(1);
3772 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3775void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3778 SDValue LHSL, LHSH, RHSL, RHSH;
3780 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3781 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3783 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3784 SDValue HiOps[3] = { LHSH, RHSH };
3786 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3787 HiOps[2] =
Lo.getValue(1);
3788 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3792 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3795void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3803 unsigned CarryOp, NoCarryOp;
3805 switch(
N->getOpcode()) {
3825 SDValue LHSL, LHSH, RHSL, RHSH;
3826 GetExpandedInteger(LHS, LHSL, LHSH);
3827 GetExpandedInteger(RHS, RHSL, RHSH);
3829 SDValue LoOps[2] = { LHSL, RHSL };
3830 SDValue HiOps[3] = { LHSH, RHSH };
3832 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3833 HiOps[2] =
Lo.getValue(1);
3834 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3836 Ovf =
Hi.getValue(1);
3841 SplitInteger(Sum,
Lo,
Hi);
3852 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3863 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3869 SDValue LHSL, LHSH, RHSL, RHSH;
3871 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3872 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3874 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3877 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3878 HiOps[2] =
Lo.getValue(1);
3879 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3883 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3886void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3889 SDValue LHSL, LHSH, RHSL, RHSH;
3891 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3892 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3898 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3899 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3903 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3906void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3911 if (
Op.getValueType().bitsLE(NVT)) {
3918 assert(getTypeAction(
Op.getValueType()) ==
3920 "Only know how to promote this result!");
3923 "Operand over promoted?");
3925 SplitInteger(Res,
Lo,
Hi);
3929void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3932 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3933 EVT NVT =
Lo.getValueType();
3934 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3938 if (NVTBits < EVTBits) {
3941 EVTBits - NVTBits)));
3951void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3954 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3955 EVT NVT =
Lo.getValueType();
3956 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3960 if (NVTBits < EVTBits) {
3963 EVTBits - NVTBits)));
3971void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3974 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3979void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3982 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3991 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3992 EVT NVT =
Lo.getValueType();
3998void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
4002 auto Constant = cast<ConstantSDNode>(
N);
4004 bool IsTarget =
Constant->isTargetOpcode();
4005 bool IsOpaque =
Constant->isOpaque();
4016 GetExpandedInteger(N0,
Lo,
Hi);
4017 EVT NVT =
Lo.getValueType();
4048 EVT VT =
N->getValueType(0);
4052 SplitInteger(Neg, NegLo, NegHi);
4060void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
4064 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4065 EVT NVT =
Lo.getValueType();
4082 SplitInteger(Result,
Lo,
Hi);
4087 EVT VT =
N->getValueType(0);
4091 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4093 LC = RTLIB::CTPOP_I32;
4094 else if (VT == MVT::i64)
4095 LC = RTLIB::CTPOP_I64;
4096 else if (VT == MVT::i128)
4097 LC = RTLIB::CTPOP_I128;
4099 "LibCall explicitly requested, but not available");
4109 GetExpandedInteger(
Op,
Lo,
Hi);
4110 EVT NVT =
Lo.getValueType();
4116void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
4120 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4121 EVT NVT =
Lo.getValueType();
4150 ReplaceValueWith(
SDValue(
N, 1), Chain);
4158 Chain =
Op.getValue(1);
4167 EVT VT =
N->getValueType(0);
4171 bool IsStrict =
N->isStrictFPOpcode();
4173 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
4175 Op = GetPromotedFloat(
Op);
4179 Op.getValueType() == MVT::bf16) {
4185 EVT OpVT =
Op.getValueType();
4189 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
4195 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
4196 CallOptions, dl, Chain);
4197 SplitInteger(Tmp.first,
Lo,
Hi);
4200 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
4206 SplitInteger(Res,
Lo,
Hi);
4212 bool IsStrict =
N->isStrictFPOpcode();
4213 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
4217 "Input type needs to be promoted!");
4219 EVT VT =
Op.getValueType();
4221 if (VT == MVT::f16) {
4227 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4231 LC = RTLIB::LROUND_F32;
4232 else if (VT == MVT::f64)
4233 LC = RTLIB::LROUND_F64;
4234 else if (VT == MVT::f80)
4235 LC = RTLIB::LROUND_F80;
4236 else if (VT == MVT::f128)
4237 LC = RTLIB::LROUND_F128;
4238 else if (VT == MVT::ppcf128)
4239 LC = RTLIB::LROUND_PPCF128;
4240 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
4244 LC = RTLIB::LRINT_F32;
4245 else if (VT == MVT::f64)
4246 LC = RTLIB::LRINT_F64;
4247 else if (VT == MVT::f80)
4248 LC = RTLIB::LRINT_F80;
4249 else if (VT == MVT::f128)
4250 LC = RTLIB::LRINT_F128;
4251 else if (VT == MVT::ppcf128)
4252 LC = RTLIB::LRINT_PPCF128;
4253 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
4257 LC = RTLIB::LLROUND_F32;
4258 else if (VT == MVT::f64)
4259 LC = RTLIB::LLROUND_F64;
4260 else if (VT == MVT::f80)
4261 LC = RTLIB::LLROUND_F80;
4262 else if (VT == MVT::f128)
4263 LC = RTLIB::LLROUND_F128;
4264 else if (VT == MVT::ppcf128)
4265 LC = RTLIB::LLROUND_PPCF128;
4266 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
4270 LC = RTLIB::LLRINT_F32;
4271 else if (VT == MVT::f64)
4272 LC = RTLIB::LLRINT_F64;
4273 else if (VT == MVT::f80)
4274 LC = RTLIB::LLRINT_F80;
4275 else if (VT == MVT::f128)
4276 LC = RTLIB::LLRINT_F128;
4277 else if (VT == MVT::ppcf128)
4278 LC = RTLIB::LLRINT_PPCF128;
4279 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
4283 EVT RetVT =
N->getValueType(0);
4287 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
4288 Op, CallOptions, dl,
4290 SplitInteger(Tmp.first,
Lo,
Hi);
4292 if (
N->isStrictFPOpcode())
4293 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
4296void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
4298 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
4301 ExpandRes_NormalLoad(
N,
Lo,
Hi);
4307 EVT VT =
N->getValueType(0);
4318 if (
N->getMemoryVT().bitsLE(NVT)) {
4319 EVT MemVT =
N->getMemoryVT();
4322 N->getBaseAlign(), MMOFlags, AAInfo);
4325 Ch =
Lo.getValue(1);
4330 unsigned LoSize =
Lo.getValueSizeInBits();
4344 Lo = DAG.
getLoad(NVT, dl, Ch,
Ptr,
N->getPointerInfo(),
N->getBaseAlign(),
4347 unsigned ExcessBits =
4355 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4356 N->getBaseAlign(), MMOFlags, AAInfo);
4365 EVT MemVT =
N->getMemoryVT();
4368 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4374 N->getBaseAlign(), MMOFlags, AAInfo);
4380 N->getPointerInfo().getWithOffset(IncrementSize),
4382 N->getBaseAlign(), MMOFlags, AAInfo);
4406 ReplaceValueWith(
SDValue(
N, 1), Ch);
4409void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4413 GetExpandedInteger(
N->getOperand(0), LL, LH);
4414 GetExpandedInteger(
N->getOperand(1), RL, RH);
4418 Flags.setDisjoint(
N->getFlags().hasDisjoint());
4424void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4426 EVT VT =
N->getValueType(0);
4431 GetExpandedInteger(
N->getOperand(0), LL, LH);
4432 GetExpandedInteger(
N->getOperand(1), RL, RH);
4440 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4442 LC = RTLIB::MUL_I16;
4443 else if (VT == MVT::i32)
4444 LC = RTLIB::MUL_I32;
4445 else if (VT == MVT::i64)
4446 LC = RTLIB::MUL_I64;
4447 else if (VT == MVT::i128)
4448 LC = RTLIB::MUL_I128;
4459 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4462 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4474 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4479 SplitInteger(Result,
Lo,
Hi);
4485 SplitInteger(Result,
Lo,
Hi);
4491 SplitInteger(Result,
Lo,
Hi);
4502 EVT VT =
N->getValueType(0);
4506 uint64_t Scale =
N->getConstantOperandVal(2);
4518 EVT BoolVT = getSetCCResultType(VT);
4543 SplitInteger(Result,
Lo,
Hi);
4549 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4553 GetExpandedInteger(LHS, LL, LH);
4554 GetExpandedInteger(RHS, RL, RH);
4558 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4566 SplitInteger(LoTmp, Result[0], Result[1]);
4567 SplitInteger(HiTmp, Result[2], Result[3]);
4569 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4572 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4573 "the size of the current value type");
4595 if (Scale % NVTSize) {
4611 if (Scale == VTSize)
4632 EVT BoolNVT = getSetCCResultType(NVT);
4635 if (Scale < NVTSize) {
4642 }
else if (Scale == NVTSize) {
4645 }
else if (Scale < VTSize) {
4653 "(and saturation can't happen with Scale==VTSize).");
4660 if (Scale < NVTSize) {
4665 unsigned OverflowBits = VTSize - Scale + 1;
4666 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4667 "Extent of overflow bits must start within HL");
4684 }
else if (Scale == NVTSize) {
4697 }
else if (Scale < VTSize) {
4700 unsigned OverflowBits = VTSize - Scale + 1;
4727 N->getConstantOperandVal(2), DAG);
4731 N->getConstantOperandVal(2), TLI, DAG);
4732 SplitInteger(Res,
Lo,
Hi);
4735void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4738 "Node has unexpected Opcode");
4753 SDValue LHSL, LHSH, RHSL, RHSH;
4754 GetExpandedInteger(LHS, LHSL, LHSH);
4755 GetExpandedInteger(RHS, RHSL, RHSH);
4759 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4761 Ovf =
Hi.getValue(1);
4768 SplitInteger(Sum,
Lo,
Hi);
4793 EVT VT =
LHS.getValueType();
4796 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4800 EVT OType =
Node->getValueType(1);
4808void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4810 EVT VT =
N->getValueType(0);
4812 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4820 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4822 LC = RTLIB::SDIV_I16;
4823 else if (VT == MVT::i32)
4824 LC = RTLIB::SDIV_I32;
4825 else if (VT == MVT::i64)
4826 LC = RTLIB::SDIV_I64;
4827 else if (VT == MVT::i128)
4828 LC = RTLIB::SDIV_I128;
4829 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4833 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4836void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4839 SDValue Shiftee =
N->getOperand(0);
4852 "Shifting unit is not a a power of two!");
4854 const bool IsOneStepShift =
4860 if (!IsOneStepShift)
4864 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4865 unsigned VTByteWidth = VTBitWidth / 8;
4867 "Shiftee type size is not a power of two!");
4868 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4869 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4882 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4887 unsigned WideningOpc =
4889 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4896 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackAlign);
4902 Flags.setExact(IsOneStepShift);
4922 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4924 WillIndexUpwards = !WillIndexUpwards;
4927 if (WillIndexUpwards) {
4931 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4932 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4941 DAG.
getLoad(VT, dl, Ch, AdjStackPtr,
4946 if (!IsOneStepShift) {
4949 DAG.
getConstant(ShiftUnitInBits - 1, dl, ShAmtVT));
4950 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4954 SplitInteger(Res,
Lo,
Hi);
4957void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4959 EVT VT =
N->getValueType(0);
4960 unsigned Opc =
N->getOpcode();
4966 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4970 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4989 const bool LegalOrCustom =
4993 unsigned ExpansionFactor = 1;
4995 for (
EVT TmpVT = NVT;;) {
4997 if (NewTMPVT == TmpVT)
5007 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
5009 if (LegalOrCustom &&
5013 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
5019 SDValue ShiftOp =
N->getOperand(1);
5024 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
5026 Hi =
Lo.getValue(1);
5031 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5036 LC = RTLIB::SHL_I16;
5037 else if (VT == MVT::i32)
5038 LC = RTLIB::SHL_I32;
5039 else if (VT == MVT::i64)
5040 LC = RTLIB::SHL_I64;
5041 else if (VT == MVT::i128)
5042 LC = RTLIB::SHL_I128;
5046 LC = RTLIB::SRL_I16;
5047 else if (VT == MVT::i32)
5048 LC = RTLIB::SRL_I32;
5049 else if (VT == MVT::i64)
5050 LC = RTLIB::SRL_I64;
5051 else if (VT == MVT::i128)
5052 LC = RTLIB::SRL_I128;
5057 LC = RTLIB::SRA_I16;
5058 else if (VT == MVT::i32)
5059 LC = RTLIB::SRA_I32;
5060 else if (VT == MVT::i64)
5061 LC = RTLIB::SRA_I64;
5062 else if (VT == MVT::i128)
5063 LC = RTLIB::SRA_I128;
5070 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
5073 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5077 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
5081void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
5086 if (
Op.getValueType().bitsLE(NVT)) {
5097 assert(getTypeAction(
Op.getValueType()) ==
5099 "Only know how to promote this result!");
5102 "Operand over promoted?");
5104 SplitInteger(Res,
Lo,
Hi);
5112void DAGTypeLegalizer::
5115 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5116 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
5138void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
5140 EVT VT =
N->getValueType(0);
5142 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5150 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5152 LC = RTLIB::SREM_I16;
5153 else if (VT == MVT::i32)
5154 LC = RTLIB::SREM_I32;
5155 else if (VT == MVT::i64)
5156 LC = RTLIB::SREM_I64;
5157 else if (VT == MVT::i128)
5158 LC = RTLIB::SREM_I128;
5159 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
5163 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5166void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
5178void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
5180 EVT VT =
N->getValueType(0);
5199 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
5200 GetExpandedInteger(LHS, LHSLow, LHSHigh);
5201 GetExpandedInteger(RHS, RHSLow, RHSHigh);
5203 EVT BitVT =
N->getValueType(1);
5228 SplitInteger(Three,
Lo,
Hi);
5232 ReplaceValueWith(
SDValue(
N, 1), Overflow);
5241 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5243 LC = RTLIB::MULO_I32;
5244 else if (VT == MVT::i64)
5245 LC = RTLIB::MULO_I64;
5246 else if (VT == MVT::i128)
5247 LC = RTLIB::MULO_I128;
5256 N->getOperand(1), MulLo, MulHi);
5262 SplitInteger(MulLo,
Lo,
Hi);
5263 ReplaceValueWith(
SDValue(
N, 1), Overflow);
5275 EVT ArgVT =
Op.getValueType();
5278 Entry.IsSExt =
true;
5279 Entry.IsZExt =
false;
5280 Args.push_back(Entry);
5286 Entry.IsSExt =
true;
5287 Entry.IsZExt =
false;
5288 Args.push_back(Entry);
5307 ReplaceValueWith(
SDValue(
N, 1), Ofl);
5310void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
5312 EVT VT =
N->getValueType(0);
5314 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5323 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5326 if (isTypeLegal(NVT)) {
5328 GetExpandedInteger(
N->getOperand(0), InL, InH);
5338 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5340 LC = RTLIB::UDIV_I16;
5341 else if (VT == MVT::i32)
5342 LC = RTLIB::UDIV_I32;
5343 else if (VT == MVT::i64)
5344 LC = RTLIB::UDIV_I64;
5345 else if (VT == MVT::i128)
5346 LC = RTLIB::UDIV_I128;
5347 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
5350 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5353void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
5355 EVT VT =
N->getValueType(0);
5357 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5366 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5369 if (isTypeLegal(NVT)) {
5371 GetExpandedInteger(
N->getOperand(0), InL, InH);
5381 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5383 LC = RTLIB::UREM_I16;
5384 else if (VT == MVT::i32)
5385 LC = RTLIB::UREM_I32;
5386 else if (VT == MVT::i64)
5387 LC = RTLIB::UREM_I64;
5388 else if (VT == MVT::i128)
5389 LC = RTLIB::UREM_I128;
5390 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
5393 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5396void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
5401 if (
Op.getValueType().bitsLE(NVT)) {
5408 assert(getTypeAction(
Op.getValueType()) ==
5410 "Only know how to promote this result!");
5413 "Operand over promoted?");
5415 SplitInteger(Res,
Lo,
Hi);
5423void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5426 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5431 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5432 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5438void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5443 SplitInteger(Res,
Lo,
Hi);
5446void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5452 N->getOperand(0),
N->getOperand(1));
5453 SplitInteger(Res,
Lo,
Hi);
5460 GetExpandedInteger(
N->getOperand(0), In3, In4);
5461 GetExpandedInteger(
N->getOperand(1), In1, In2);
5465 unsigned Opc =
N->getOpcode();
5468 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5491 EVT VT =
N->getValueType(0);
5501 SplitInteger(Res,
Lo,
Hi);
5508 "cannot use llvm.read_register with illegal type", Fn,
N->getDebugLoc()));
5509 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
5524bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5528 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5531 switch (
N->getOpcode()) {
5534 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5535 N->dump(&DAG);
dbgs() <<
"\n";
5540 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5544 Res = ExpandOp_FAKE_USE(
N);
5548 case ISD::EXPERIMENTAL_VP_SPLAT:
5551 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5557 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5564 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5569 case ISD::UCMP: Res = ExpandIntOp_CMP(
N);
break;
5573 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5576 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5578 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5579 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5580 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5583 Res = ExpandIntOp_WRITE_REGISTER(
N, OpNo);
5588 if (!Res.
getNode())
return false;
5596 "Invalid operand expansion");
5598 ReplaceValueWith(
SDValue(
N, 0), Res);
5604void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5608 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5609 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5610 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5630 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5631 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5663 RHSLo, LowCC,
false, DagCombineInfo, dl);
5670 RHSHi, CCCode,
false, DagCombineInfo, dl);
5683 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5685 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5694 if (LHSHi == RHSHi) {
5707 if (HasSETCCCARRY) {
5710 bool FlipOperands =
false;
5738 false, DagCombineInfo, dl);
5747 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5748 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5749 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5761 N->getOperand(4)), 0);
5765 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5766 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5767 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5778 N->getOperand(2),
N->getOperand(3),
5783 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5784 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5785 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5790 "Unexpected setcc expansion!");
5806 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5807 GetExpandedInteger(LHS, LHSLo, LHSHi);
5808 GetExpandedInteger(RHS, RHSLo, RHSHi);
5821 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5831 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5844 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5849 bool IsStrict =
N->isStrictFPOpcode();
5853 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5854 EVT DstVT =
N->getValueType(0);
5857 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5858 "Don't know how to expand this XINT_TO_FP!");
5861 std::pair<SDValue, SDValue> Tmp =
5867 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5868 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5873 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5876 return ExpandOp_NormalStore(
N, OpNo);
5879 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5881 EVT VT =
N->getOperand(1).getValueType();
5892 if (
N->getMemoryVT().bitsLE(NVT)) {
5893 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5895 N->getMemoryVT(),
N->getBaseAlign(), MMOFlags,
5901 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5906 unsigned ExcessBits =
5914 N->getPointerInfo().getWithOffset(IncrementSize),
5915 NEVT,
N->getBaseAlign(), MMOFlags, AAInfo);
5921 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5923 EVT ExtVT =
N->getMemoryVT();
5926 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5944 N->getBaseAlign(), MMOFlags, AAInfo);
5950 N->getPointerInfo().getWithOffset(IncrementSize),
5952 N->getBaseAlign(), MMOFlags, AAInfo);
5958 GetExpandedInteger(
N->getOperand(0), InL, InH);
5967 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5968 cast<AtomicSDNode>(
N)->getMemOperand());
5972SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5973 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5974 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5978 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5983SDValue DAGTypeLegalizer::ExpandIntOp_WRITE_REGISTER(
SDNode *
N,
unsigned OpNo) {
5986 "cannot use llvm.write_register with illegal type", Fn,
5989 return N->getOperand(0);
5992SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5995 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5996 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6002SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
6004 unsigned Factor =
N->getNumOperands();
6007 for (
unsigned i = 0; i != Factor; i++)
6008 Ops[i] = GetPromotedInteger(
N->getOperand(i));
6013 for (
unsigned i = 0; i != Factor; i++)
6019SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
6021 EVT OutVT =
N->getValueType(0);
6023 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6027 SDValue BaseIdx =
N->getOperand(1);
6054 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
6063 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
6067 "Promoted operand has an element type greater than result");
6080 InOp0 = GetPromotedInteger(InOp0);
6088 for (
unsigned i = 0; i != OutNumElems; ++i) {
6093 N->getOperand(0), Index);
6102SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
6103 EVT OutVT =
N->getValueType(0);
6105 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6117 Vec = GetPromotedInteger(Vec);
6123SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
6126 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6132SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
6134 EVT VT =
N->getValueType(0);
6139 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6140 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6147 EVT OutVT =
N->getValueType(0);
6149 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6150 unsigned NumElems =
N->getNumOperands();
6158 for (
unsigned i = 0; i != NumElems; ++i) {
6160 EVT OpVT =
Op.getValueType();
6165 if (OpVT.
bitsLT(NOutVTElem)) {
6171 ExtOpc = NOutExtOpc;
6184 assert(!
N->getOperand(0).getValueType().isVector() &&
6185 "Input must be a scalar");
6187 EVT OutVT =
N->getValueType(0);
6189 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6193 if (
N->isVPOpcode())
6194 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op,
N->getOperand(1),
6197 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
6202 EVT OutVT =
N->getValueType(0);
6205 "Type must be promoted to a scalable vector type");
6206 const APInt &StepVal =
N->getConstantOperandAPInt(0);
6211SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
6214 EVT OutVT =
N->getValueType(0);
6216 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6218 unsigned NumOperands =
N->getNumOperands();
6223 SDUse *MaxSizedValue = std::max_element(
6225 EVT AVT = A.getValueType().getVectorElementType();
6226 EVT BVT = B.getValueType().getVectorElementType();
6227 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
6233 for (
unsigned I = 0;
I < NumOperands; ++
I) {
6235 EVT OpVT =
Op.getValueType();
6237 Op = GetPromotedInteger(
Op);
6240 "Unhandled legalization type");
6257 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
6258 assert(NumElem * NumOperands == NumOutElem &&
6259 "Unexpected number of elements");
6263 for (
unsigned i = 0; i < NumOperands; ++i) {
6266 Op = GetPromotedInteger(
Op);
6267 EVT SclrTy =
Op.getValueType().getVectorElementType();
6268 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
6269 "Unexpected number of elements");
6271 for (
unsigned j = 0;
j < NumElem; ++
j) {
6281SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
6282 EVT VT =
N->getValueType(0);
6284 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
6292 if (getTypeAction(
N->getOperand(0).getValueType())
6296 switch(
N->getOpcode()) {
6298 Promoted = SExtPromotedInteger(
N->getOperand(0));
6301 Promoted = ZExtPromotedInteger(
N->getOperand(0));
6304 Promoted = GetPromotedInteger(
N->getOperand(0));
6309 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
6313 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
6316SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(
SDNode *
N) {
6317 EVT VT =
N->getValueType(0);
6322SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(
SDNode *
N) {
6323 EVT VT =
N->getValueType(0);
6328SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(
SDNode *
N) {
6330 EVT VT =
N->getValueType(0);
6332 SDValue ExtAcc = GetPromotedInteger(
N->getOperand(0));
6333 return DAG.
getNode(
N->getOpcode(),
DL, NVT, ExtAcc,
N->getOperand(1),
6337SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
6338 EVT OutVT =
N->getValueType(0);
6340 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6345 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6348 NOutVTElem,
N->getOperand(1));
6350 V0, ConvElem,
N->getOperand(2));
6358 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
6366 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
6367 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
6368 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
6375 assert(
N->getNumValues() == 3 &&
"Expected 3 values for PATCHPOINT");
6389SDValue DAGTypeLegalizer::PromoteIntRes_READ_REGISTER(
SDNode *
N) {
6392 "cannot use llvm.read_register with illegal type", Fn,
N->getDebugLoc()));
6395 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
6399SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
6401 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6413SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
6418 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6438 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0), V0, VPromoted);
6441SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
6443 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6446 N->getValueType(0).getVectorNumElements());
6451SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
6454 EVT ResVT =
N->getValueType(0);
6455 unsigned NumElems =
N->getNumOperands();
6462 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
6470 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
6476 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
6478 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
6479 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
6481 for (
unsigned i=0; i<NumElem; ++i) {
6493SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
6506 for (
unsigned I = 0;
I < OpNo;
I++)
6509 EVT Ty =
Op.getValueType();
6521 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6526 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6532SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6545 for (
unsigned I = 0;
I < OpNo;
I++)
6548 EVT Ty =
Op.getValueType();
6560 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6565 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
MachineInstr unsigned OpIdx
const SmallVectorImpl< MachineOperand > & Cond
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
unsigned getActiveBits() const
Compute the number of active bits in the value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
LLVM_ABI SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
LLVM_ABI SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
LLVM_ABI void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
LLVM_ABI std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
virtual unsigned getPreferredFPToIntOpcode(unsigned Op, EVT FromVT, EVT ToVT) const
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, const SDValue LHS, const SDValue RHS, SDValue &Lo, SDValue &Hi) const
Calculate full product of LHS and RHS either via a libcall or through brute force expansion of the mu...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
void forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl, bool Signed, SDValue &Lo, SDValue &Hi, SDValue LHS, SDValue RHS, SDValue HiLHS=SDValue(), SDValue HiRHS=SDValue()) const
Calculate the product twice the width of LHS and RHS.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
This class is used to represent a VP_LOAD node.
This class is used to represent a VP_STORE node.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ LOOP_DEPENDENCE_RAW_MASK
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ VECTOR_FIND_LAST_ACTIVE
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ EXPERIMENTAL_VECTOR_HISTOGRAM
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
@ LOOP_DEPENDENCE_WAR_MASK
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
LLVM_ABI Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
@ Success
The lock was released successfully.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)