31#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::VP_BITREVERSE:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
79 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
80 case ISD::VP_CTTZ_ELTS:
81 Res = PromoteIntRes_VP_CttzElements(
N);
84 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
85 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
86 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
88 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
94 Res = PromoteIntRes_Select(
N);
99 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
101 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
103 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
106 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
108 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
110 case ISD::VP_SRA: Res = PromoteIntRes_SRA(
N);
break;
112 case ISD::VP_SRL: Res = PromoteIntRes_SRL(
N);
break;
113 case ISD::VP_TRUNCATE:
115 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
116 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
120 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
122 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
124 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
126 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
128 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
131 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
134 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
136 Res = PromoteIntRes_BUILD_VECTOR(
N);
140 Res = PromoteIntRes_ScalarOp(
N);
144 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
149 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
152 case ISD::VP_SIGN_EXTEND:
154 case ISD::VP_ZERO_EXTEND:
157 case ISD::VP_FP_TO_SINT:
158 case ISD::VP_FP_TO_UINT:
166 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
170 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
174 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
189 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
198 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
207 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
210 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
212 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
214 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
230 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
232 case ISD::VP_SADDSAT:
233 case ISD::VP_UADDSAT:
234 case ISD::VP_SSUBSAT:
235 case ISD::VP_USUBSAT:
236 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
241 Res = PromoteIntRes_CMP(
N);
254 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
257 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
271 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
275 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
287 Res = PromoteIntRes_VECREDUCE(
N);
290 case ISD::VP_REDUCE_ADD:
291 case ISD::VP_REDUCE_MUL:
292 case ISD::VP_REDUCE_AND:
293 case ISD::VP_REDUCE_OR:
294 case ISD::VP_REDUCE_XOR:
295 case ISD::VP_REDUCE_SMAX:
296 case ISD::VP_REDUCE_SMIN:
297 case ISD::VP_REDUCE_UMAX:
298 case ISD::VP_REDUCE_UMIN:
299 Res = PromoteIntRes_VP_REDUCE(
N);
303 Res = PromoteIntRes_FREEZE(
N);
308 Res = PromoteIntRes_Rotate(
N);
313 Res = PromoteIntRes_FunnelShift(
N);
318 Res = PromoteIntRes_VPFunnelShift(
N);
322 Res = PromoteIntRes_IS_FPCLASS(
N);
325 Res = PromoteIntRes_FFREXP(
N);
330 Res = PromoteIntRes_XRINT(
N);
336 SetPromotedInteger(
SDValue(
N, ResNo), Res);
341 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
342 return GetPromotedInteger(
Op);
347 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
349 Op.getValueType(),
Op,
N->getOperand(1));
354 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
356 Op.getValueType(),
Op,
N->getOperand(1));
362 N->getMemoryVT(), ResVT,
363 N->getChain(),
N->getBasePtr(),
382 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
392 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
395 N->getChain(),
N->getBasePtr(),
396 Op2,
N->getMemOperand());
407 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
418 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
428 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
431 Op2 = SExtPromotedInteger(Op2);
434 Op2 = ZExtPromotedInteger(Op2);
437 Op2 = GetPromotedInteger(Op2);
446 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
447 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
449 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
458 EVT OutVT =
N->getValueType(0);
462 switch (getTypeAction(InVT)) {
489 BitConvertToInteger(GetScalarizedVector(InOp)));
498 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
499 Lo = BitConvertToInteger(
Lo);
500 Hi = BitConvertToInteger(
Hi);
508 JoinIntegers(
Lo,
Hi));
542 if (isTypeLegal(WideOutVT)) {
543 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
553 CreateStackStoreLoad(InOp, OutVT));
557 SDValue V = GetPromotedInteger(
N->getOperand(0));
559 V.getValueType(), V);
563 SDValue Op = GetPromotedInteger(
N->getOperand(0));
564 EVT OVT =
N->getValueType(0);
565 EVT NVT =
Op.getValueType();
585 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
586 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
591 SDValue Op = GetPromotedInteger(
N->getOperand(0));
592 EVT OVT =
N->getValueType(0);
593 EVT NVT =
Op.getValueType();
613 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
614 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
623 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
628 EVT VT =
N->getValueType(0);
637 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
642 EVT OVT =
N->getValueType(0);
661 if (!
N->isVPOpcode()) {
663 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
672 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
673 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
674 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
675 ExtractLeadingBits, Mask, EVL);
679 EVT OVT =
N->getValueType(0);
696 if (!
N->isVPOpcode()) {
697 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
703 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
709 SDValue Op = GetPromotedInteger(
N->getOperand(0));
710 EVT OVT =
N->getValueType(0);
711 EVT NVT =
Op.getValueType();
729 unsigned NewOpc =
N->getOpcode();
730 if (NewOpc ==
ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
742 N->getOperand(1),
N->getOperand(2));
743 NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
746 if (!
N->isVPOpcode())
748 return DAG.
getNode(NewOpc, dl, NVT,
Op,
N->getOperand(1),
N->getOperand(2));
751SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(
SDNode *
N) {
754 return DAG.
getNode(
N->getOpcode(),
DL, NewVT,
N->ops());
757SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
772 EVT SVT =
In.getValueType().getScalarType();
784 unsigned NewOpc =
N->getOpcode();
801 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
804 NewOpc = ISD::VP_FP_TO_SINT;
807 if (
N->isStrictFPOpcode()) {
808 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
809 {
N->getOperand(0),
N->getOperand(1)});
813 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
814 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
817 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
829 N->getOpcode() == ISD::VP_FP_TO_UINT)
836SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
840 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
844SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
848 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
851SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
856 N->getOperand(0),
N->getOperand(1));
864 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
872 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
884 if (getTypeAction(
N->getOperand(0).getValueType())
886 SDValue Res = GetPromotedInteger(
N->getOperand(0));
904 if (
N->getNumOperands() != 1) {
905 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
906 assert(
N->isVPOpcode() &&
"Expected VP opcode");
907 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
908 N->getOperand(1),
N->getOperand(2));
910 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
920 N->getMemoryVT(),
N->getMemOperand());
930 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
938 N->getOffset(),
N->getMask(), ExtPassThru,
939 N->getMemoryVT(),
N->getMemOperand(),
940 N->getAddressingMode(), ExtType,
941 N->isExpandingLoad());
950 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
952 "Gather result type and the passThru argument type should be the same");
959 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
960 N->getIndex(),
N->getScale() };
962 N->getMemoryVT(), dl, Ops,
963 N->getMemOperand(),
N->getIndexType(),
976 EVT VT =
N->getValueType(0);
977 EVT SVT = getSetCCResultType(VT);
978 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
979 unsigned NumOps =
N->getNumOperands();
980 assert(NumOps <= 3 &&
"Too many operands");
982 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
990 ReplaceValueWith(
SDValue(
N, 0), Res);
996template <
class MatchContextClass>
1008 MatchContextClass matcher(DAG, TLI,
N);
1011 unsigned Opcode = matcher.getRootBaseOpcode();
1015 SDValue Op1Promoted, Op2Promoted;
1017 Op1Promoted = GetPromotedInteger(Op1);
1018 Op2Promoted = ZExtPromotedInteger(Op2);
1020 Op1Promoted = ZExtPromotedInteger(Op1);
1021 Op2Promoted = ZExtPromotedInteger(Op2);
1023 Op1Promoted = SExtPromotedInteger(Op1);
1024 Op2Promoted = SExtPromotedInteger(Op2);
1033 matcher.getNode(
ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1034 return matcher.getNode(
ISD::UMIN, dl, PromotedType,
Add, SatMax);
1039 return matcher.getNode(
ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1044 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1057 "addition, subtraction or left shift");
1060 unsigned SHLAmount = NewBits - OldBits;
1067 matcher.
getNode(
ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1070 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1071 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1080 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1081 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1082 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1089 SDValue Op1Promoted, Op2Promoted;
1095 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1096 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1098 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1099 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1101 EVT OldType =
N->getOperand(0).getValueType();
1115 Op2Promoted,
N->getOperand(2));
1117 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1120 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1125 unsigned SatW,
bool Signed,
1128 EVT VT = V.getValueType();
1155 EVT VT =
LHS.getValueType();
1173 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1179 "Tried to saturate to more than the original type?");
1188 SDValue Op1Promoted, Op2Promoted;
1194 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1195 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1197 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1198 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1201 unsigned Scale =
N->getConstantOperandVal(2);
1210 N->getValueType(0).getScalarSizeInBits();
1215 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1216 Op2Promoted,
N->getOperand(2));
1226 Op2Promoted, Scale, DAG)) {
1229 N->getValueType(0).getScalarSizeInBits(),
1237 N->getValueType(0).getScalarSizeInBits());
1240SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1242 return PromoteIntRes_Overflow(
N);
1246 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1247 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1248 EVT OVT =
N->getOperand(0).getValueType();
1249 EVT NVT =
LHS.getValueType();
1264 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1270 EVT PromotedResultTy =
1273 N->getOperand(0),
N->getOperand(1));
1279 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1280 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1282 unsigned Opcode =
N->getOpcode();
1283 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1290 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1291 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1293 LHS.getValueType(),
N->getOperand(0),
1294 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1298 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1299 EVT InVT =
N->getOperand(OpNo).getValueType();
1302 EVT SVT = getSetCCResultType(InVT);
1310 SVT = getSetCCResultType(InVT);
1318 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1319 "Vector compare must return a vector result!");
1323 if (
N->isStrictFPOpcode()) {
1325 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1326 N->getOperand(2),
N->getOperand(3)};
1327 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1332 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1333 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1349 EVT VT =
N->getValueType(0);
1355 ReplaceValueWith(
SDValue(
N, 0), Res);
1360 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1362 if (
N->getOpcode() != ISD::VP_SHL) {
1364 RHS = ZExtPromotedInteger(RHS);
1372 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1377SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1378 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1380 Op.getValueType(),
Op,
N->getOperand(1));
1383SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1387 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1388 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1389 if (
N->getNumOperands() == 2)
1391 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1392 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1394 N->getOperand(2),
N->getOperand(3));
1398 if (
N->getNumOperands() == 2) {
1400 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1401 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1404 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1405 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1409 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1410 SDValue RHS = VPSExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1416 if (
N->getNumOperands() == 2) {
1418 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1419 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1422 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1423 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1427 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1428 SDValue RHS = VPZExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1439 SExtOrZExtPromotedOperands(LHS, RHS);
1442 LHS.getValueType(), LHS, RHS);
1447 if (
N->getOpcode() != ISD::VP_SRA) {
1449 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1451 RHS = ZExtPromotedInteger(RHS);
1458 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1460 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1467 if (
N->getOpcode() != ISD::VP_SRL) {
1469 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1471 RHS = ZExtPromotedInteger(RHS);
1478 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1480 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1488 ReplaceValueWith(
SDValue(
N, 0), Res);
1493 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1494 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1497 Amt = ZExtPromotedInteger(Amt);
1501 EVT OldVT =
N->getOperand(0).getValueType();
1502 EVT VT =
Lo.getValueType();
1503 unsigned Opcode =
N->getOpcode();
1517 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1542SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1543 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1544 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1549 Amt = VPZExtPromotedInteger(Amt, Mask, EVL);
1553 EVT OldVT =
N->getOperand(0).getValueType();
1554 EVT VT =
Lo.getValueType();
1555 unsigned Opcode =
N->getOpcode();
1556 bool IsFSHR = Opcode == ISD::VP_FSHR;
1561 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1569 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1572 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1575 Res = DAG.
getNode(IsFSHR ? ISD::VP_SRL : ISD::VP_SHL,
DL, VT, Res, Amt,
1578 Res = DAG.
getNode(ISD::VP_SRL,
DL, VT, Res, HiShift, Mask, EVL);
1584 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1589 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1607 Res = GetPromotedInteger(InOp);
1614 "Dst and Src must have the same number of elements");
1616 "Promoted vector type must be a power of two");
1619 GetSplitVector(InOp, EOp1, EOp2);
1627 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1628 "Expected VP_TRUNCATE opcode");
1629 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1630 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1631 std::tie(EVLLo, EVLHi) =
1632 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1633 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1634 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1641 SDValue WideInOp = GetWidenedVector(InOp);
1646 N->getValueType(0).getScalarType(), NumElem);
1661 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1662 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1667SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1669 return PromoteIntRes_Overflow(
N);
1673 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1674 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1675 EVT OVT =
N->getOperand(0).getValueType();
1676 EVT NVT =
LHS.getValueType();
1690 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1701 return PromoteIntRes_Overflow(
N);
1713 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1714 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1716 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1720 LHS, RHS,
N->getOperand(2));
1730 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1731 return PromoteIntRes_Overflow(
N);
1735 EVT OVT =
N->getValueType(0);
1748 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1752SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1755 return PromoteIntRes_Overflow(
N);
1759 EVT SmallVT =
LHS.getValueType();
1766 LHS = SExtPromotedInteger(LHS);
1767 RHS = SExtPromotedInteger(RHS);
1769 LHS = ZExtPromotedInteger(LHS);
1770 RHS = ZExtPromotedInteger(RHS);
1801 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1807 N->getValueType(0)));
1813 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1820 EVT VT =
N->getValueType(0);
1828 for (
unsigned i = 0; i < NumRegs; ++i) {
1829 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1830 N->getConstantOperandVal(3));
1836 std::reverse(Parts.begin(), Parts.end());
1841 for (
unsigned i = 1; i < NumRegs; ++i) {
1852 ReplaceValueWith(
SDValue(
N, 1), Chain);
1865bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1868 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1873 switch (
N->getOpcode()) {
1876 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1877 N->dump(&DAG);
dbgs() <<
"\n";
1883 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1886 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1887 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1893 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1897 Res = PromoteIntOp_ScalarOp(
N);
1900 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1903 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1905 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
1906 case ISD::VP_SINT_TO_FP:
1909 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1911 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1913 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1915 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1917 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1919 case ISD::VP_TRUNCATE:
1923 case ISD::VP_UINT_TO_FP:
1928 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
1936 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1939 case ISD::UCMP: Res = PromoteIntOp_CMP(
N);
break;
1942 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1968 case ISD::VP_REDUCE_ADD:
1969 case ISD::VP_REDUCE_MUL:
1970 case ISD::VP_REDUCE_AND:
1971 case ISD::VP_REDUCE_OR:
1972 case ISD::VP_REDUCE_XOR:
1973 case ISD::VP_REDUCE_SMAX:
1974 case ISD::VP_REDUCE_SMIN:
1975 case ISD::VP_REDUCE_UMAX:
1976 case ISD::VP_REDUCE_UMIN:
1977 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
1982 Res = PromoteIntOp_STACKMAP(
N, OpNo);
1985 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
1987 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1988 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1989 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
1991 case ISD::EXPERIMENTAL_VP_SPLICE:
1992 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
1997 if (!Res.
getNode())
return false;
2004 const bool IsStrictFp =
N->isStrictFPOpcode();
2006 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2007 "Invalid operand expansion");
2011 ReplaceValueWith(
SDValue(
N, 0), Res);
2021void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
2022 SDValue OpL = GetPromotedInteger(LHS);
2023 SDValue OpR = GetPromotedInteger(RHS);
2029 unsigned OpLEffectiveBits =
2031 unsigned OpREffectiveBits =
2033 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2034 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2041 LHS = SExtPromotedInteger(LHS);
2042 RHS = SExtPromotedInteger(RHS);
2053 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2054 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2061 LHS = ZExtPromotedInteger(LHS);
2062 RHS = ZExtPromotedInteger(RHS);
2067void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2076 LHS = SExtPromotedInteger(LHS);
2077 RHS = SExtPromotedInteger(RHS);
2082 "Unknown integer comparison!");
2084 SExtOrZExtPromotedOperands(LHS, RHS);
2088 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2093 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2095 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2101 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
2104SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2105 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2109 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2114 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2118SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2119 assert(OpNo == 1 &&
"only know how to promote condition");
2122 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2126 N->getOperand(2)), 0);
2131 EVT OVT =
N->getOperand(0).getValueType();
2132 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2133 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2134 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2147 EVT VecVT =
N->getValueType(0);
2150 "Legal vector of one illegal element?");
2155 assert(
N->getOperand(0).getValueSizeInBits() >=
2156 N->getValueType(0).getScalarSizeInBits() &&
2157 "Type of inserted value narrower than vector element type!");
2160 for (
unsigned i = 0; i < NumElts; ++i)
2161 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2166SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2173 assert(
N->getOperand(1).getValueSizeInBits() >=
2174 N->getValueType(0).getScalarSizeInBits() &&
2175 "Type of inserted value narrower than vector element type!");
2177 GetPromotedInteger(
N->getOperand(1)),
2182 assert(OpNo == 2 &&
"Different operand and result vector types?");
2188 N->getOperand(1),
Idx), 0);
2195 GetPromotedInteger(
N->getOperand(0))), 0);
2198SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2199 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2201 EVT OpTy =
N->getOperand(1).getValueType();
2204 if (
SDValue Res = WidenVSELECTMask(
N))
2206 Res,
N->getOperand(1),
N->getOperand(2));
2210 Cond = PromoteTargetBoolean(
Cond, OpVT);
2213 N->getOperand(2)), 0);
2216SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2217 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2221 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2225 N->getOperand(3),
N->getOperand(4)), 0);
2228SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2229 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2233 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2239 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2242 N->getOperand(3),
N->getOperand(4)),
2248 ZExtPromotedInteger(
N->getOperand(1))), 0);
2253 ? ZExtPromotedInteger(
N->getOperand(0))
2254 : SExtPromotedInteger(
N->getOperand(0));
2256 ? ZExtPromotedInteger(
N->getOperand(1))
2257 : SExtPromotedInteger(
N->getOperand(1));
2264 ZExtPromotedInteger(
N->getOperand(2))), 0);
2268 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2275SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2277 EVT VT =
N->getValueType(0);
2278 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2280 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2288 return DAG.
getNode(ISD::VP_SRA, dl, VT, Shl, ShAmt,
N->getOperand(1),
2293 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2295 SExtPromotedInteger(
N->getOperand(0)),
2296 N->getOperand(1),
N->getOperand(2)),
2299 SExtPromotedInteger(
N->getOperand(0))), 0);
2302SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2304 SExtPromotedInteger(
N->getOperand(1))), 0);
2312 SDValue Val = GetPromotedInteger(
N->getValue());
2316 N->getMemoryVT(),
N->getMemOperand());
2327 Mask = PromoteTargetBoolean(Mask, DataVT);
2333 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2334 DataOp = GetPromotedInteger(DataOp);
2337 N->getOffset(), Mask,
N->getMemoryVT(),
2338 N->getMemOperand(),
N->getAddressingMode(),
2339 true,
N->isCompressingStore());
2344 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2345 EVT DataVT =
N->getValueType(0);
2346 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2348 NewOps[OpNo] =
Mask;
2365 EVT DataVT =
N->getValueType(0);
2366 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2367 }
else if (OpNo == 4) {
2369 if (
N->isIndexSigned())
2371 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2373 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2375 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2389 bool TruncateStore =
N->isTruncatingStore();
2394 EVT DataVT =
N->getValue().getValueType();
2395 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2396 }
else if (OpNo == 4) {
2398 if (
N->isIndexSigned())
2400 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2402 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2404 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2405 TruncateStore =
true;
2409 SDLoc(
N), NewOps,
N->getMemOperand(),
2410 N->getIndexType(), TruncateStore);
2414 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2415 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2417 N->getOperand(1),
N->getOperand(2));
2422 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2424 ZExtPromotedInteger(
N->getOperand(0)),
2425 N->getOperand(1),
N->getOperand(2)),
2428 ZExtPromotedInteger(
N->getOperand(0))), 0);
2431SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2433 ZExtPromotedInteger(
N->getOperand(1))), 0);
2438 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2443SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2445 EVT VT =
N->getValueType(0);
2446 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2448 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2451 N->getOperand(0).getValueType());
2455 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2460SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2462 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2467 bool IsStrict =
N->isStrictFPOpcode();
2479 SDValue Op = SExtPromotedInteger(
N->getOperand(1));
2489 unsigned OpOffset = IsStrict ? 1 : 0;
2492 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2493 "POWI exponent should match with sizeof(int) when doing the libcall.");
2496 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2497 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2498 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2499 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2501 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2506 switch (
N->getOpcode()) {
2514 case ISD::VP_REDUCE_ADD:
2515 case ISD::VP_REDUCE_MUL:
2516 case ISD::VP_REDUCE_AND:
2517 case ISD::VP_REDUCE_OR:
2518 case ISD::VP_REDUCE_XOR:
2522 case ISD::VP_REDUCE_SMAX:
2523 case ISD::VP_REDUCE_SMIN:
2527 case ISD::VP_REDUCE_UMAX:
2528 case ISD::VP_REDUCE_UMIN:
2538 return GetPromotedInteger(V);
2540 return SExtPromotedInteger(V);
2542 return ZExtPromotedInteger(V);
2548 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2550 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2551 EVT InVT =
Op.getValueType();
2553 EVT ResVT =
N->getValueType(0);
2554 unsigned Opcode =
N->getOpcode();
2574 Op = ZExtPromotedInteger(
N->getOperand(0));
2577 Op = SExtPromotedInteger(
N->getOperand(0));
2593 Op = ZExtPromotedInteger(
N->getOperand(0));
2596 Op = SExtPromotedInteger(
N->getOperand(0));
2610SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2617 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2621 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2623 Op = PromoteIntOpVectorReduction(
N,
Op);
2627 EVT VT =
N->getValueType(0);
2628 EVT EltVT =
Op.getValueType().getScalarType();
2643 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2647SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2650 SDValue Operand =
N->getOperand(OpNo);
2656SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2659 SDValue Operand =
N->getOperand(OpNo);
2665SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2666 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2667 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2670 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2675SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2679 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2683 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2685 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2697void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2703 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2706 switch (
N->getOpcode()) {
2709 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2710 N->dump(&DAG);
dbgs() <<
"\n";
2756 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2783 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2784 SplitInteger(Tmp.first,
Lo,
Hi);
2785 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2793 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2802 SplitInteger(Tmp,
Lo,
Hi);
2881 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2886 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2890 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2900std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2901 unsigned Opc =
Node->getOpcode();
2907 EVT RetVT =
Node->getValueType(0);
2915 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2916 "Unexpected atomic op or value type!");
2920 Node->getOperand(0));
2925void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2930 GetExpandedInteger(
N->getOperand(0), InL, InH);
2941 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2945 if (Amt.
uge(VTBits)) {
2947 }
else if (Amt.
ugt(NVTBits)) {
2951 }
else if (Amt == NVTBits) {
2968 if (Amt.
uge(VTBits)) {
2970 }
else if (Amt.
ugt(NVTBits)) {
2974 }
else if (Amt == NVTBits) {
2991 if (Amt.
uge(VTBits)) {
2994 }
else if (Amt.
ugt(NVTBits)) {
2999 }
else if (Amt == NVTBits) {
3019bool DAGTypeLegalizer::
3021 unsigned Opc =
N->getOpcode();
3029 "Expanded integer type size not a power of two!");
3036 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
3041 GetExpandedInteger(In, InL, InH);
3095 Lo = DAG.
getNode(Opc, dl, NVT, InL, Amt);
3108bool DAGTypeLegalizer::
3115 "Expanded integer type size not a power of two!");
3120 GetExpandedInteger(
N->getOperand(0), InL, InH);
3132 switch (
N->getOpcode()) {
3147 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3163 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3179 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3200void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3209 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3210 unsigned NumHalfBits = NumBits / 2;
3213 SDValue LHSL, LHSH, RHSL, RHSH;
3214 GetExpandedInteger(LHS, LHSL, LHSH);
3215 GetExpandedInteger(RHS, RHSL, RHSH);
3228 SDValue LHSL, LHSH, RHSL, RHSH;
3229 GetExpandedInteger(LHS, LHSL, LHSH);
3230 GetExpandedInteger(RHS, RHSL, RHSH);
3232 EVT CCT = getSetCCResultType(NVT);
3241 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3245 const APInt *RHSVal =
nullptr;
3246 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3247 RHSVal = &RHSConst->getAPIntValue();
3254 SDValue LHSL, LHSH, RHSL, RHSH;
3255 GetExpandedInteger(LHS, LHSL, LHSH);
3256 GetExpandedInteger(RHS, RHSL, RHSH);
3258 EVT CCT = getSetCCResultType(NVT);
3264 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3283 switch (
N->getOpcode()) {
3310 EVT VT =
N->getValueType(0);
3311 EVT CCT = getSetCCResultType(VT);
3314 SplitInteger(Result,
Lo,
Hi);
3319 SplitInteger(ExpandedCMP,
Lo,
Hi);
3322void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3326 SDValue LHSL, LHSH, RHSL, RHSH;
3327 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3328 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3331 SDValue LoOps[2] = { LHSL, RHSL };
3332 SDValue HiOps[3] = { LHSH, RHSH };
3341 HiOps[2] =
Lo.getValue(1);
3347 HiOps[2] =
Lo.getValue(1);
3369 HiOps[2] =
Lo.getValue(1);
3373 HiOps[2] =
Lo.getValue(1);
3386 EVT OvfVT = getSetCCResultType(NVT);
3426 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3429 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3432 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3450 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3464void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3467 SDValue LHSL, LHSH, RHSL, RHSH;
3469 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3470 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3472 SDValue LoOps[2] = { LHSL, RHSL };
3473 SDValue HiOps[3] = { LHSH, RHSH };
3477 HiOps[2] =
Lo.getValue(1);
3481 HiOps[2] =
Lo.getValue(1);
3487 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3490void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3493 SDValue LHSL, LHSH, RHSL, RHSH;
3495 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3496 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3498 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3499 SDValue HiOps[3] = { LHSH, RHSH };
3501 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3502 HiOps[2] =
Lo.getValue(1);
3503 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3507 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3510void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3518 unsigned CarryOp, NoCarryOp;
3520 switch(
N->getOpcode()) {
3540 SDValue LHSL, LHSH, RHSL, RHSH;
3541 GetExpandedInteger(LHS, LHSL, LHSH);
3542 GetExpandedInteger(RHS, RHSL, RHSH);
3544 SDValue LoOps[2] = { LHSL, RHSL };
3545 SDValue HiOps[3] = { LHSH, RHSH };
3547 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3548 HiOps[2] =
Lo.getValue(1);
3549 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3551 Ovf =
Hi.getValue(1);
3556 SplitInteger(Sum,
Lo,
Hi);
3567 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3578 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3584 SDValue LHSL, LHSH, RHSL, RHSH;
3586 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3587 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3589 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3592 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3593 HiOps[2] =
Lo.getValue(1);
3594 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3598 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3601void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3604 SDValue LHSL, LHSH, RHSL, RHSH;
3606 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3607 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3613 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3614 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3618 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3621void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3626 if (
Op.getValueType().bitsLE(NVT)) {
3633 assert(getTypeAction(
Op.getValueType()) ==
3635 "Only know how to promote this result!");
3638 "Operand over promoted?");
3640 SplitInteger(Res,
Lo,
Hi);
3644void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3647 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3648 EVT NVT =
Lo.getValueType();
3649 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3653 if (NVTBits < EVTBits) {
3656 EVTBits - NVTBits)));
3666void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3669 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3670 EVT NVT =
Lo.getValueType();
3671 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3675 if (NVTBits < EVTBits) {
3678 EVTBits - NVTBits)));
3686void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3689 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3694void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3697 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3706 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3707 EVT NVT =
Lo.getValueType();
3713void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3717 auto Constant = cast<ConstantSDNode>(
N);
3719 bool IsTarget =
Constant->isTargetOpcode();
3720 bool IsOpaque =
Constant->isOpaque();
3731 GetExpandedInteger(N0,
Lo,
Hi);
3732 EVT NVT =
Lo.getValueType();
3763 EVT VT =
N->getValueType(0);
3767 SplitInteger(Neg, NegLo, NegHi);
3775void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3779 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3780 EVT NVT =
Lo.getValueType();
3795void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3799 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3800 EVT NVT =
Lo.getValueType();
3806void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3810 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3811 EVT NVT =
Lo.getValueType();
3840 ReplaceValueWith(
SDValue(
N, 1), Chain);
3848 Chain =
Op.getValue(1);
3857 EVT VT =
N->getValueType(0);
3861 bool IsStrict =
N->isStrictFPOpcode();
3863 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3865 Op = GetPromotedFloat(
Op);
3868 EVT OFPVT =
Op.getValueType();
3870 Op = GetSoftPromotedHalf(
Op);
3874 SplitInteger(
Op,
Lo,
Hi);
3878 if (
Op.getValueType() == MVT::bf16) {
3885 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
3888 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3889 CallOptions, dl, Chain);
3890 SplitInteger(Tmp.first,
Lo,
Hi);
3893 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3899 SplitInteger(Res,
Lo,
Hi);
3905 bool IsStrict =
N->isStrictFPOpcode();
3906 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3910 "Input type needs to be promoted!");
3912 EVT VT =
Op.getValueType();
3914 if (VT == MVT::f16) {
3924 LC = RTLIB::LROUND_F32;
3925 else if (VT == MVT::f64)
3926 LC = RTLIB::LROUND_F64;
3927 else if (VT == MVT::f80)
3928 LC = RTLIB::LROUND_F80;
3929 else if (VT == MVT::f128)
3930 LC = RTLIB::LROUND_F128;
3931 else if (VT == MVT::ppcf128)
3932 LC = RTLIB::LROUND_PPCF128;
3933 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3937 LC = RTLIB::LRINT_F32;
3938 else if (VT == MVT::f64)
3939 LC = RTLIB::LRINT_F64;
3940 else if (VT == MVT::f80)
3941 LC = RTLIB::LRINT_F80;
3942 else if (VT == MVT::f128)
3943 LC = RTLIB::LRINT_F128;
3944 else if (VT == MVT::ppcf128)
3945 LC = RTLIB::LRINT_PPCF128;
3946 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
3950 LC = RTLIB::LLROUND_F32;
3951 else if (VT == MVT::f64)
3952 LC = RTLIB::LLROUND_F64;
3953 else if (VT == MVT::f80)
3954 LC = RTLIB::LLROUND_F80;
3955 else if (VT == MVT::f128)
3956 LC = RTLIB::LLROUND_F128;
3957 else if (VT == MVT::ppcf128)
3958 LC = RTLIB::LLROUND_PPCF128;
3959 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3963 LC = RTLIB::LLRINT_F32;
3964 else if (VT == MVT::f64)
3965 LC = RTLIB::LLRINT_F64;
3966 else if (VT == MVT::f80)
3967 LC = RTLIB::LLRINT_F80;
3968 else if (VT == MVT::f128)
3969 LC = RTLIB::LLRINT_F128;
3970 else if (VT == MVT::ppcf128)
3971 LC = RTLIB::LLRINT_PPCF128;
3972 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
3976 EVT RetVT =
N->getValueType(0);
3980 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
3981 Op, CallOptions, dl,
3983 SplitInteger(Tmp.first,
Lo,
Hi);
3985 if (
N->isStrictFPOpcode())
3986 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3989void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
3991 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
3994 ExpandRes_NormalLoad(
N,
Lo,
Hi);
4000 EVT VT =
N->getValueType(0);
4011 if (
N->getMemoryVT().bitsLE(NVT)) {
4012 EVT MemVT =
N->getMemoryVT();
4015 N->getOriginalAlign(), MMOFlags, AAInfo);
4018 Ch =
Lo.getValue(1);
4023 unsigned LoSize =
Lo.getValueSizeInBits();
4038 N->getOriginalAlign(), MMOFlags, AAInfo);
4040 unsigned ExcessBits =
4048 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4049 N->getOriginalAlign(), MMOFlags, AAInfo);
4058 EVT MemVT =
N->getMemoryVT();
4061 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4067 N->getOriginalAlign(), MMOFlags, AAInfo);
4073 N->getPointerInfo().getWithOffset(IncrementSize),
4075 N->getOriginalAlign(), MMOFlags, AAInfo);
4099 ReplaceValueWith(
SDValue(
N, 1), Ch);
4102void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4106 GetExpandedInteger(
N->getOperand(0), LL, LH);
4107 GetExpandedInteger(
N->getOperand(1), RL, RH);
4112void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4114 EVT VT =
N->getValueType(0);
4119 GetExpandedInteger(
N->getOperand(0), LL, LH);
4120 GetExpandedInteger(
N->getOperand(1), RL, RH);
4130 LC = RTLIB::MUL_I16;
4131 else if (VT == MVT::i32)
4132 LC = RTLIB::MUL_I32;
4133 else if (VT == MVT::i64)
4134 LC = RTLIB::MUL_I64;
4135 else if (VT == MVT::i128)
4136 LC = RTLIB::MUL_I128;
4148 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4151 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4163 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4168 SplitInteger(Result,
Lo,
Hi);
4174 SplitInteger(Result,
Lo,
Hi);
4180 SplitInteger(Result,
Lo,
Hi);
4191 EVT VT =
N->getValueType(0);
4195 uint64_t Scale =
N->getConstantOperandVal(2);
4207 EVT BoolVT = getSetCCResultType(VT);
4232 SplitInteger(Result,
Lo,
Hi);
4238 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4242 GetExpandedInteger(LHS, LL, LH);
4243 GetExpandedInteger(RHS, RL, RH);
4247 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4255 SplitInteger(LoTmp, Result[0], Result[1]);
4256 SplitInteger(HiTmp, Result[2], Result[3]);
4258 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4261 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4262 "the size of the current value type");
4284 if (Scale % NVTSize) {
4300 if (Scale == VTSize)
4321 EVT BoolNVT = getSetCCResultType(NVT);
4324 if (Scale < NVTSize) {
4331 }
else if (Scale == NVTSize) {
4334 }
else if (Scale < VTSize) {
4342 "(and saturation can't happen with Scale==VTSize).");
4349 if (Scale < NVTSize) {
4354 unsigned OverflowBits = VTSize - Scale + 1;
4355 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4356 "Extent of overflow bits must start within HL");
4373 }
else if (Scale == NVTSize) {
4386 }
else if (Scale < VTSize) {
4389 unsigned OverflowBits = VTSize - Scale + 1;
4416 N->getConstantOperandVal(2), DAG);
4420 N->getConstantOperandVal(2), TLI, DAG);
4421 SplitInteger(Res,
Lo,
Hi);
4424void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4427 "Node has unexpected Opcode");
4442 SDValue LHSL, LHSH, RHSL, RHSH;
4443 GetExpandedInteger(LHS, LHSL, LHSH);
4444 GetExpandedInteger(RHS, RHSL, RHSH);
4448 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4450 Ovf =
Hi.getValue(1);
4457 SplitInteger(Sum,
Lo,
Hi);
4482 EVT VT =
LHS.getValueType();
4485 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4489 EVT OType =
Node->getValueType(1);
4497void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4499 EVT VT =
N->getValueType(0);
4501 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4511 LC = RTLIB::SDIV_I16;
4512 else if (VT == MVT::i32)
4513 LC = RTLIB::SDIV_I32;
4514 else if (VT == MVT::i64)
4515 LC = RTLIB::SDIV_I64;
4516 else if (VT == MVT::i128)
4517 LC = RTLIB::SDIV_I128;
4518 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4522 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4525void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4528 SDValue Shiftee =
N->getOperand(0);
4535 bool ShiftByByteMultiple =
4540 if (!ShiftByByteMultiple)
4544 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4545 unsigned VTByteWidth = VTBitWidth / 8;
4547 "Shiftee type size is not a power of two!");
4548 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4549 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4555 Align StackSlotAlignment(1);
4563 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4568 unsigned WideningOpc =
4570 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4577 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4583 if (ShiftByByteMultiple)
4584 Flags.setExact(
true);
4596 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4598 WillIndexUpwards = !WillIndexUpwards;
4601 if (WillIndexUpwards) {
4605 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4606 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4615 VT, dl, Ch, AdjStackPtr,
4620 if (!ShiftByByteMultiple) {
4623 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4627 SplitInteger(Res,
Lo,
Hi);
4630void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4632 EVT VT =
N->getValueType(0);
4633 unsigned Opc =
N->getOpcode();
4639 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4643 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4662 const bool LegalOrCustom =
4666 unsigned ExpansionFactor = 1;
4668 for (
EVT TmpVT = NVT;;) {
4670 if (NewTMPVT == TmpVT)
4680 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
4682 if (LegalOrCustom &&
4686 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4692 SDValue ShiftOp =
N->getOperand(1);
4697 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4699 Hi =
Lo.getValue(1);
4709 LC = RTLIB::SHL_I16;
4710 else if (VT == MVT::i32)
4711 LC = RTLIB::SHL_I32;
4712 else if (VT == MVT::i64)
4713 LC = RTLIB::SHL_I64;
4714 else if (VT == MVT::i128)
4715 LC = RTLIB::SHL_I128;
4719 LC = RTLIB::SRL_I16;
4720 else if (VT == MVT::i32)
4721 LC = RTLIB::SRL_I32;
4722 else if (VT == MVT::i64)
4723 LC = RTLIB::SRL_I64;
4724 else if (VT == MVT::i128)
4725 LC = RTLIB::SRL_I128;
4730 LC = RTLIB::SRA_I16;
4731 else if (VT == MVT::i32)
4732 LC = RTLIB::SRA_I32;
4733 else if (VT == MVT::i64)
4734 LC = RTLIB::SRA_I64;
4735 else if (VT == MVT::i128)
4736 LC = RTLIB::SRA_I128;
4743 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4746 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4750 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
4754void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4759 if (
Op.getValueType().bitsLE(NVT)) {
4770 assert(getTypeAction(
Op.getValueType()) ==
4772 "Only know how to promote this result!");
4775 "Operand over promoted?");
4777 SplitInteger(Res,
Lo,
Hi);
4785void DAGTypeLegalizer::
4788 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4789 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4811void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4813 EVT VT =
N->getValueType(0);
4815 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4825 LC = RTLIB::SREM_I16;
4826 else if (VT == MVT::i32)
4827 LC = RTLIB::SREM_I32;
4828 else if (VT == MVT::i64)
4829 LC = RTLIB::SREM_I64;
4830 else if (VT == MVT::i128)
4831 LC = RTLIB::SREM_I128;
4832 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4836 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4839void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4851void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
4853 EVT VT =
N->getValueType(0);
4872 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4873 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4874 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4876 EVT BitVT =
N->getValueType(1);
4901 SplitInteger(Three,
Lo,
Hi);
4905 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4916 LC = RTLIB::MULO_I32;
4917 else if (VT == MVT::i64)
4918 LC = RTLIB::MULO_I64;
4919 else if (VT == MVT::i128)
4920 LC = RTLIB::MULO_I128;
4933 SplitInteger(
Mul, MulLo, MulHi);
4939 SplitInteger(MulLo,
Lo,
Hi);
4940 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4953 EVT ArgVT =
Op.getValueType();
4957 Entry.IsSExt =
true;
4958 Entry.IsZExt =
false;
4959 Args.push_back(Entry);
4965 Entry.IsSExt =
true;
4966 Entry.IsZExt =
false;
4967 Args.push_back(Entry);
4986 ReplaceValueWith(
SDValue(
N, 1), Ofl);
4989void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
4991 EVT VT =
N->getValueType(0);
4993 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5002 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5005 if (isTypeLegal(NVT)) {
5007 GetExpandedInteger(
N->getOperand(0), InL, InH);
5019 LC = RTLIB::UDIV_I16;
5020 else if (VT == MVT::i32)
5021 LC = RTLIB::UDIV_I32;
5022 else if (VT == MVT::i64)
5023 LC = RTLIB::UDIV_I64;
5024 else if (VT == MVT::i128)
5025 LC = RTLIB::UDIV_I128;
5026 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
5029 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5032void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
5034 EVT VT =
N->getValueType(0);
5036 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5045 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5048 if (isTypeLegal(NVT)) {
5050 GetExpandedInteger(
N->getOperand(0), InL, InH);
5062 LC = RTLIB::UREM_I16;
5063 else if (VT == MVT::i32)
5064 LC = RTLIB::UREM_I32;
5065 else if (VT == MVT::i64)
5066 LC = RTLIB::UREM_I64;
5067 else if (VT == MVT::i128)
5068 LC = RTLIB::UREM_I128;
5069 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
5072 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5075void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
5080 if (
Op.getValueType().bitsLE(NVT)) {
5087 assert(getTypeAction(
Op.getValueType()) ==
5089 "Only know how to promote this result!");
5092 "Operand over promoted?");
5094 SplitInteger(Res,
Lo,
Hi);
5102void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5105 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5110 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5111 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5117void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5122 SplitInteger(Res,
Lo,
Hi);
5125void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5131 N->getOperand(0),
N->getOperand(1));
5132 SplitInteger(Res,
Lo,
Hi);
5139 GetExpandedInteger(
N->getOperand(0), In3, In4);
5140 GetExpandedInteger(
N->getOperand(1), In1, In2);
5144 unsigned Opc =
N->getOpcode();
5147 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5164 Lo = DAG.
getNode(Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5165 Hi = DAG.
getNode(Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5170 EVT VT =
N->getValueType(0);
5180 SplitInteger(Res,
Lo,
Hi);
5191bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5195 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5198 switch (
N->getOpcode()) {
5201 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5202 N->dump(&DAG);
dbgs() <<
"\n";
5207 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5214 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5220 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5227 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5232 case ISD::UCMP: Res = ExpandIntOp_CMP(
N);
break;
5236 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5239 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5241 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5242 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5243 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5248 if (!Res.
getNode())
return false;
5256 "Invalid operand expansion");
5258 ReplaceValueWith(
SDValue(
N, 0), Res);
5264void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5268 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5269 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5270 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5290 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5291 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5323 RHSLo, LowCC,
false, DagCombineInfo, dl);
5330 RHSHi, CCCode,
false, DagCombineInfo, dl);
5343 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5345 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5354 if (LHSHi == RHSHi) {
5367 if (HasSETCCCARRY) {
5370 bool FlipOperands =
false;
5398 false, DagCombineInfo, dl);
5407 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5408 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5409 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5421 N->getOperand(4)), 0);
5425 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5426 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5427 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5438 N->getOperand(2),
N->getOperand(3),
5443 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5444 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5445 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5450 "Unexpected setcc expansion!");
5466 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5467 GetExpandedInteger(LHS, LHSLo, LHSHi);
5468 GetExpandedInteger(RHS, RHSLo, RHSHi);
5481 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5491 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5504 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5509 bool IsStrict =
N->isStrictFPOpcode();
5513 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5514 EVT DstVT =
N->getValueType(0);
5517 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5518 "Don't know how to expand this XINT_TO_FP!");
5521 std::pair<SDValue, SDValue> Tmp =
5527 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5528 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5533 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5536 return ExpandOp_NormalStore(
N, OpNo);
5539 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5541 EVT VT =
N->getOperand(1).getValueType();
5552 if (
N->getMemoryVT().bitsLE(NVT)) {
5553 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5555 N->getMemoryVT(),
N->getOriginalAlign(), MMOFlags,
5561 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5564 N->getOriginalAlign(), MMOFlags, AAInfo);
5566 unsigned ExcessBits =
5574 N->getPointerInfo().getWithOffset(IncrementSize),
5575 NEVT,
N->getOriginalAlign(), MMOFlags, AAInfo);
5581 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5583 EVT ExtVT =
N->getMemoryVT();
5586 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5604 N->getOriginalAlign(), MMOFlags, AAInfo);
5610 N->getPointerInfo().getWithOffset(IncrementSize),
5612 N->getOriginalAlign(), MMOFlags, AAInfo);
5618 GetExpandedInteger(
N->getOperand(0), InL, InH);
5627 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5628 cast<AtomicSDNode>(
N)->getMemOperand());
5632SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5633 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5634 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5638 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5643SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5646 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5647 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5653SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
5656 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5657 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5666SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
5668 EVT OutVT =
N->getValueType(0);
5670 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5674 SDValue BaseIdx =
N->getOperand(1);
5701 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5710 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5714 "Promoted operand has an element type greater than result");
5727 InOp0 = GetPromotedInteger(
N->getOperand(0));
5734 for (
unsigned i = 0; i != OutNumElems; ++i) {
5750SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
5751 EVT OutVT =
N->getValueType(0);
5753 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5765 Vec = GetPromotedInteger(Vec);
5771SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
5774 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5780SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
5782 EVT VT =
N->getValueType(0);
5787 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5788 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5795 EVT OutVT =
N->getValueType(0);
5797 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5798 unsigned NumElems =
N->getNumOperands();
5806 for (
unsigned i = 0; i != NumElems; ++i) {
5808 EVT OpVT =
Op.getValueType();
5813 if (OpVT.
bitsLT(NOutVTElem)) {
5819 ExtOpc = NOutExtOpc;
5832 assert(!
N->getOperand(0).getValueType().isVector() &&
5833 "Input must be a scalar");
5835 EVT OutVT =
N->getValueType(0);
5837 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5842 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
5847 EVT OutVT =
N->getValueType(0);
5850 "Type must be promoted to a scalable vector type");
5851 const APInt &StepVal =
N->getConstantOperandAPInt(0);
5856SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
5859 EVT OutVT =
N->getValueType(0);
5861 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5863 unsigned NumOperands =
N->getNumOperands();
5868 SDUse *MaxSizedValue = std::max_element(
5870 EVT AVT = A.getValueType().getVectorElementType();
5871 EVT BVT = B.getValueType().getVectorElementType();
5872 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5878 for (
unsigned I = 0;
I < NumOperands; ++
I) {
5880 EVT OpVT =
Op.getValueType();
5882 Op = GetPromotedInteger(
Op);
5885 "Unhandled legalization type");
5902 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
5903 assert(NumElem * NumOperands == NumOutElem &&
5904 "Unexpected number of elements");
5908 for (
unsigned i = 0; i < NumOperands; ++i) {
5911 Op = GetPromotedInteger(
Op);
5912 EVT SclrTy =
Op.getValueType().getVectorElementType();
5913 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
5914 "Unexpected number of elements");
5916 for (
unsigned j = 0;
j < NumElem; ++
j) {
5926SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
5927 EVT VT =
N->getValueType(0);
5929 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
5937 if (getTypeAction(
N->getOperand(0).getValueType())
5941 switch(
N->getOpcode()) {
5943 Promoted = SExtPromotedInteger(
N->getOperand(0));
5946 Promoted = ZExtPromotedInteger(
N->getOperand(0));
5949 Promoted = GetPromotedInteger(
N->getOperand(0));
5954 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
5958 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
5961SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
5962 EVT OutVT =
N->getValueType(0);
5964 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5969 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5972 NOutVTElem,
N->getOperand(1));
5974 V0, ConvElem,
N->getOperand(2));
5982 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
5990 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
5991 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
5992 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
5995SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
5997 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6009SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
6014 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6024SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
6026 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6029 N->getValueType(0).getVectorNumElements());
6034SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
6037 EVT ResVT =
N->getValueType(0);
6038 unsigned NumElems =
N->getNumOperands();
6043 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
6045 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
6053 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
6059 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
6061 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
6062 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
6064 for (
unsigned i=0; i<NumElem; ++i) {
6076SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
6089 for (
unsigned I = 0;
I < OpNo;
I++)
6092 EVT Ty =
Op.getValueType();
6104 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6109 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6115SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6128 for (
unsigned I = 0;
I < OpNo;
I++)
6131 EVT Ty =
Op.getValueType();
6143 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6148 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)