31#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::VP_BITREVERSE:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
79 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
80 case ISD::VP_CTTZ_ELTS:
81 Res = PromoteIntRes_VP_CttzElements(
N);
84 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
85 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
86 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
88 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
94 Res = PromoteIntRes_Select(
N);
99 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
101 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
103 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
106 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
108 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
110 case ISD::VP_SRA: Res = PromoteIntRes_SRA(
N);
break;
112 case ISD::VP_SRL: Res = PromoteIntRes_SRL(
N);
break;
113 case ISD::VP_TRUNCATE:
115 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
116 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
120 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
122 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
124 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
126 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
128 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
131 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
134 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
136 Res = PromoteIntRes_BUILD_VECTOR(
N);
140 Res = PromoteIntRes_ScalarOp(
N);
144 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
149 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
152 case ISD::VP_SIGN_EXTEND:
154 case ISD::VP_ZERO_EXTEND:
157 case ISD::VP_FP_TO_SINT:
158 case ISD::VP_FP_TO_UINT:
166 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
170 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
174 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
189 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
198 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
207 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
210 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
212 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
214 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
230 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
232 case ISD::VP_SADDSAT:
233 case ISD::VP_UADDSAT:
234 case ISD::VP_SSUBSAT:
235 case ISD::VP_USUBSAT:
236 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
241 Res = PromoteIntRes_CMP(
N);
254 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
257 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
271 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
275 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
287 Res = PromoteIntRes_VECREDUCE(
N);
290 case ISD::VP_REDUCE_ADD:
291 case ISD::VP_REDUCE_MUL:
292 case ISD::VP_REDUCE_AND:
293 case ISD::VP_REDUCE_OR:
294 case ISD::VP_REDUCE_XOR:
295 case ISD::VP_REDUCE_SMAX:
296 case ISD::VP_REDUCE_SMIN:
297 case ISD::VP_REDUCE_UMAX:
298 case ISD::VP_REDUCE_UMIN:
299 Res = PromoteIntRes_VP_REDUCE(
N);
303 Res = PromoteIntRes_FREEZE(
N);
308 Res = PromoteIntRes_Rotate(
N);
313 Res = PromoteIntRes_FunnelShift(
N);
318 Res = PromoteIntRes_VPFunnelShift(
N);
322 Res = PromoteIntRes_IS_FPCLASS(
N);
325 Res = PromoteIntRes_FFREXP(
N);
330 Res = PromoteIntRes_XRINT(
N);
334 Res = PromoteIntRes_PATCHPOINT(
N);
340 SetPromotedInteger(
SDValue(
N, ResNo), Res);
345 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
346 return GetPromotedInteger(
Op);
351 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
353 Op.getValueType(),
Op,
N->getOperand(1));
358 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
360 Op.getValueType(),
Op,
N->getOperand(1));
366 N->getMemoryVT(), ResVT,
367 N->getChain(),
N->getBasePtr(),
386 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
396 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
399 N->getChain(),
N->getBasePtr(),
400 Op2,
N->getMemOperand());
411 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
422 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
432 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
435 Op2 = SExtPromotedInteger(Op2);
438 Op2 = ZExtPromotedInteger(Op2);
441 Op2 = GetPromotedInteger(Op2);
450 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
451 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
453 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
462 EVT OutVT =
N->getValueType(0);
466 switch (getTypeAction(InVT)) {
493 BitConvertToInteger(GetScalarizedVector(InOp)));
502 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
503 Lo = BitConvertToInteger(
Lo);
504 Hi = BitConvertToInteger(
Hi);
512 JoinIntegers(
Lo,
Hi));
546 if (isTypeLegal(WideOutVT)) {
547 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
557 CreateStackStoreLoad(InOp, OutVT));
561 SDValue V = GetPromotedInteger(
N->getOperand(0));
563 V.getValueType(), V);
567 SDValue Op = GetPromotedInteger(
N->getOperand(0));
568 EVT OVT =
N->getValueType(0);
569 EVT NVT =
Op.getValueType();
589 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
590 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
595 SDValue Op = GetPromotedInteger(
N->getOperand(0));
596 EVT OVT =
N->getValueType(0);
597 EVT NVT =
Op.getValueType();
617 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
618 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
627 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
632 EVT VT =
N->getValueType(0);
641 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
646 EVT OVT =
N->getValueType(0);
662 unsigned CtlzOpcode =
N->getOpcode();
663 if (CtlzOpcode ==
ISD::CTLZ || CtlzOpcode == ISD::VP_CTLZ) {
668 if (!
N->isVPOpcode()) {
670 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
678 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
679 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
680 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
681 ExtractLeadingBits, Mask, EVL);
684 CtlzOpcode == ISD::VP_CTLZ_ZERO_UNDEF) {
686 SDValue Op = GetPromotedInteger(
N->getOperand(0));
691 if (!
N->isVPOpcode()) {
693 return DAG.
getNode(CtlzOpcode, dl, NVT,
Op);
698 Op = DAG.
getNode(ISD::VP_SHL, dl, NVT,
Op, ShiftConst, Mask, EVL);
699 return DAG.
getNode(CtlzOpcode, dl, NVT,
Op, Mask, EVL);
705 EVT OVT =
N->getValueType(0);
722 if (!
N->isVPOpcode()) {
723 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
729 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
735 SDValue Op = GetPromotedInteger(
N->getOperand(0));
736 EVT OVT =
N->getValueType(0);
737 EVT NVT =
Op.getValueType();
755 unsigned NewOpc =
N->getOpcode();
756 if (NewOpc ==
ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
768 N->getOperand(1),
N->getOperand(2));
769 NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
772 if (!
N->isVPOpcode())
774 return DAG.
getNode(NewOpc, dl, NVT,
Op,
N->getOperand(1),
N->getOperand(2));
777SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(
SDNode *
N) {
780 return DAG.
getNode(
N->getOpcode(),
DL, NewVT,
N->ops());
783SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
798 EVT SVT =
In.getValueType().getScalarType();
810 unsigned NewOpc =
N->getOpcode();
827 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
830 NewOpc = ISD::VP_FP_TO_SINT;
833 if (
N->isStrictFPOpcode()) {
834 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
835 {
N->getOperand(0),
N->getOperand(1)});
839 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
840 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
843 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
855 N->getOpcode() == ISD::VP_FP_TO_UINT)
862SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
866 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
870SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
874 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
877SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
882 N->getOperand(0),
N->getOperand(1));
890 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
898 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
910 if (getTypeAction(
N->getOperand(0).getValueType())
912 SDValue Res = GetPromotedInteger(
N->getOperand(0));
930 if (
N->getNumOperands() != 1) {
931 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
932 assert(
N->isVPOpcode() &&
"Expected VP opcode");
933 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
934 N->getOperand(1),
N->getOperand(2));
936 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
946 N->getMemoryVT(),
N->getMemOperand());
956 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
964 N->getOffset(),
N->getMask(), ExtPassThru,
965 N->getMemoryVT(),
N->getMemOperand(),
966 N->getAddressingMode(), ExtType,
967 N->isExpandingLoad());
976 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
978 "Gather result type and the passThru argument type should be the same");
985 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
986 N->getIndex(),
N->getScale() };
988 N->getMemoryVT(), dl, Ops,
989 N->getMemOperand(),
N->getIndexType(),
1002 EVT VT =
N->getValueType(0);
1003 EVT SVT = getSetCCResultType(VT);
1004 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
1005 unsigned NumOps =
N->getNumOperands();
1006 assert(NumOps <= 3 &&
"Too many operands");
1008 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
1016 ReplaceValueWith(
SDValue(
N, 0), Res);
1022template <
class MatchContextClass>
1034 MatchContextClass matcher(DAG, TLI,
N);
1037 unsigned Opcode = matcher.getRootBaseOpcode();
1041 SDValue Op1Promoted, Op2Promoted;
1043 Op1Promoted = GetPromotedInteger(Op1);
1044 Op2Promoted = ZExtPromotedInteger(Op2);
1046 Op1Promoted = ZExtPromotedInteger(Op1);
1047 Op2Promoted = ZExtPromotedInteger(Op2);
1049 Op1Promoted = SExtPromotedInteger(Op1);
1050 Op2Promoted = SExtPromotedInteger(Op2);
1059 matcher.getNode(
ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1060 return matcher.getNode(
ISD::UMIN, dl, PromotedType,
Add, SatMax);
1065 return matcher.getNode(
ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1070 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1083 "addition, subtraction or left shift");
1086 unsigned SHLAmount = NewBits - OldBits;
1093 matcher.
getNode(
ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1096 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1097 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1106 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1107 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1108 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1115 SDValue Op1Promoted, Op2Promoted;
1121 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1122 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1124 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1125 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1127 EVT OldType =
N->getOperand(0).getValueType();
1141 Op2Promoted,
N->getOperand(2));
1143 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1146 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1151 unsigned SatW,
bool Signed,
1154 EVT VT = V.getValueType();
1181 EVT VT =
LHS.getValueType();
1199 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1205 "Tried to saturate to more than the original type?");
1214 SDValue Op1Promoted, Op2Promoted;
1220 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1221 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1223 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1224 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1227 unsigned Scale =
N->getConstantOperandVal(2);
1236 N->getValueType(0).getScalarSizeInBits();
1241 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1242 Op2Promoted,
N->getOperand(2));
1252 Op2Promoted, Scale, DAG)) {
1255 N->getValueType(0).getScalarSizeInBits(),
1263 N->getValueType(0).getScalarSizeInBits());
1266SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1268 return PromoteIntRes_Overflow(
N);
1272 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1273 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1274 EVT OVT =
N->getOperand(0).getValueType();
1275 EVT NVT =
LHS.getValueType();
1290 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1296 EVT PromotedResultTy =
1299 N->getOperand(0),
N->getOperand(1));
1305 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1306 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1308 unsigned Opcode =
N->getOpcode();
1309 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1316 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1317 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1319 LHS.getValueType(),
N->getOperand(0),
1320 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1324 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1325 EVT InVT =
N->getOperand(OpNo).getValueType();
1328 EVT SVT = getSetCCResultType(InVT);
1336 SVT = getSetCCResultType(InVT);
1344 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1345 "Vector compare must return a vector result!");
1349 if (
N->isStrictFPOpcode()) {
1351 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1352 N->getOperand(2),
N->getOperand(3)};
1353 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1358 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1359 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1375 EVT VT =
N->getValueType(0);
1381 ReplaceValueWith(
SDValue(
N, 0), Res);
1386 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1388 if (
N->getOpcode() != ISD::VP_SHL) {
1390 RHS = ZExtPromotedInteger(RHS);
1398 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1403SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1404 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1406 Op.getValueType(),
Op,
N->getOperand(1));
1409SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1413 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1414 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1415 if (
N->getNumOperands() == 2)
1417 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1418 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1420 N->getOperand(2),
N->getOperand(3));
1424 if (
N->getNumOperands() == 2) {
1426 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1427 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1430 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1431 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1435 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1436 SDValue RHS = VPSExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1442 if (
N->getNumOperands() == 2) {
1444 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1445 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1448 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1449 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1453 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1454 SDValue RHS = VPZExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1465 SExtOrZExtPromotedOperands(LHS, RHS);
1468 LHS.getValueType(), LHS, RHS);
1473 if (
N->getOpcode() != ISD::VP_SRA) {
1475 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1477 RHS = ZExtPromotedInteger(RHS);
1484 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1486 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1493 if (
N->getOpcode() != ISD::VP_SRL) {
1495 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1497 RHS = ZExtPromotedInteger(RHS);
1504 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1506 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1514 ReplaceValueWith(
SDValue(
N, 0), Res);
1519 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1520 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1523 Amt = ZExtPromotedInteger(Amt);
1527 EVT OldVT =
N->getOperand(0).getValueType();
1528 EVT VT =
Lo.getValueType();
1529 unsigned Opcode =
N->getOpcode();
1543 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1568SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1569 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1570 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1575 Amt = VPZExtPromotedInteger(Amt, Mask, EVL);
1579 EVT OldVT =
N->getOperand(0).getValueType();
1580 EVT VT =
Lo.getValueType();
1581 unsigned Opcode =
N->getOpcode();
1582 bool IsFSHR = Opcode == ISD::VP_FSHR;
1587 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1595 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1598 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1601 Res = DAG.
getNode(IsFSHR ? ISD::VP_SRL : ISD::VP_SHL,
DL, VT, Res, Amt,
1604 Res = DAG.
getNode(ISD::VP_SRL,
DL, VT, Res, HiShift, Mask, EVL);
1610 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1615 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1633 Res = GetPromotedInteger(InOp);
1640 "Dst and Src must have the same number of elements");
1642 "Promoted vector type must be a power of two");
1645 GetSplitVector(InOp, EOp1, EOp2);
1653 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1654 "Expected VP_TRUNCATE opcode");
1655 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1656 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1657 std::tie(EVLLo, EVLHi) =
1658 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1659 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1660 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1667 SDValue WideInOp = GetWidenedVector(InOp);
1672 N->getValueType(0).getScalarType(), NumElem);
1687 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1688 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1693SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1695 return PromoteIntRes_Overflow(
N);
1699 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1700 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1701 EVT OVT =
N->getOperand(0).getValueType();
1702 EVT NVT =
LHS.getValueType();
1716 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1727 return PromoteIntRes_Overflow(
N);
1739 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1740 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1742 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1746 LHS, RHS,
N->getOperand(2));
1756 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1757 return PromoteIntRes_Overflow(
N);
1761 EVT OVT =
N->getValueType(0);
1774 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1778SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1781 return PromoteIntRes_Overflow(
N);
1785 EVT SmallVT =
LHS.getValueType();
1792 LHS = SExtPromotedInteger(LHS);
1793 RHS = SExtPromotedInteger(RHS);
1795 LHS = ZExtPromotedInteger(LHS);
1796 RHS = ZExtPromotedInteger(RHS);
1827 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1833 N->getValueType(0)));
1839 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1846 EVT VT =
N->getValueType(0);
1854 for (
unsigned i = 0; i < NumRegs; ++i) {
1855 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1856 N->getConstantOperandVal(3));
1862 std::reverse(Parts.begin(), Parts.end());
1867 for (
unsigned i = 1; i < NumRegs; ++i) {
1878 ReplaceValueWith(
SDValue(
N, 1), Chain);
1891bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1894 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1899 switch (
N->getOpcode()) {
1902 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1903 N->dump(&DAG);
dbgs() <<
"\n";
1909 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1912 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1913 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1919 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1923 Res = PromoteIntOp_ScalarOp(
N);
1926 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1929 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1931 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
1932 case ISD::VP_SINT_TO_FP:
1935 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1937 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1939 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1941 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1943 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1945 case ISD::VP_TRUNCATE:
1949 case ISD::VP_UINT_TO_FP:
1954 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
1962 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1965 case ISD::UCMP: Res = PromoteIntOp_CMP(
N);
break;
1968 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1994 case ISD::VP_REDUCE_ADD:
1995 case ISD::VP_REDUCE_MUL:
1996 case ISD::VP_REDUCE_AND:
1997 case ISD::VP_REDUCE_OR:
1998 case ISD::VP_REDUCE_XOR:
1999 case ISD::VP_REDUCE_SMAX:
2000 case ISD::VP_REDUCE_SMIN:
2001 case ISD::VP_REDUCE_UMAX:
2002 case ISD::VP_REDUCE_UMIN:
2003 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
2008 Res = PromoteIntOp_STACKMAP(
N, OpNo);
2011 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
2013 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2014 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2015 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
2017 case ISD::EXPERIMENTAL_VP_SPLICE:
2018 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
2023 if (!Res.
getNode())
return false;
2030 const bool IsStrictFp =
N->isStrictFPOpcode();
2032 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2033 "Invalid operand expansion");
2037 ReplaceValueWith(
SDValue(
N, 0), Res);
2047void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
2048 SDValue OpL = GetPromotedInteger(LHS);
2049 SDValue OpR = GetPromotedInteger(RHS);
2055 unsigned OpLEffectiveBits =
2057 unsigned OpREffectiveBits =
2059 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2060 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2067 LHS = SExtPromotedInteger(LHS);
2068 RHS = SExtPromotedInteger(RHS);
2079 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2080 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2087 LHS = ZExtPromotedInteger(LHS);
2088 RHS = ZExtPromotedInteger(RHS);
2093void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2102 LHS = SExtPromotedInteger(LHS);
2103 RHS = SExtPromotedInteger(RHS);
2108 "Unknown integer comparison!");
2110 SExtOrZExtPromotedOperands(LHS, RHS);
2114 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2119 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2121 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2127 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
2130SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2131 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2135 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2140 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2144SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2145 assert(OpNo == 1 &&
"only know how to promote condition");
2148 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2152 N->getOperand(2)), 0);
2157 EVT OVT =
N->getOperand(0).getValueType();
2158 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2159 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2160 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2173 EVT VecVT =
N->getValueType(0);
2176 "Legal vector of one illegal element?");
2181 assert(
N->getOperand(0).getValueSizeInBits() >=
2182 N->getValueType(0).getScalarSizeInBits() &&
2183 "Type of inserted value narrower than vector element type!");
2186 for (
unsigned i = 0; i < NumElts; ++i)
2187 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2192SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2199 assert(
N->getOperand(1).getValueSizeInBits() >=
2200 N->getValueType(0).getScalarSizeInBits() &&
2201 "Type of inserted value narrower than vector element type!");
2203 GetPromotedInteger(
N->getOperand(1)),
2208 assert(OpNo == 2 &&
"Different operand and result vector types?");
2214 N->getOperand(1),
Idx), 0);
2221 GetPromotedInteger(
N->getOperand(0))), 0);
2224SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2225 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2227 EVT OpTy =
N->getOperand(1).getValueType();
2230 if (
SDValue Res = WidenVSELECTMask(
N))
2232 Res,
N->getOperand(1),
N->getOperand(2));
2236 Cond = PromoteTargetBoolean(
Cond, OpVT);
2239 N->getOperand(2)), 0);
2242SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2243 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2247 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2251 N->getOperand(3),
N->getOperand(4)), 0);
2254SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2255 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2259 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2265 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2268 N->getOperand(3),
N->getOperand(4)),
2274 ZExtPromotedInteger(
N->getOperand(1))), 0);
2279 ? ZExtPromotedInteger(
N->getOperand(0))
2280 : SExtPromotedInteger(
N->getOperand(0));
2282 ? ZExtPromotedInteger(
N->getOperand(1))
2283 : SExtPromotedInteger(
N->getOperand(1));
2290 ZExtPromotedInteger(
N->getOperand(2))), 0);
2294 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2301SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2303 EVT VT =
N->getValueType(0);
2304 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2306 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2314 return DAG.
getNode(ISD::VP_SRA, dl, VT, Shl, ShAmt,
N->getOperand(1),
2319 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2321 SExtPromotedInteger(
N->getOperand(0)),
2322 N->getOperand(1),
N->getOperand(2)),
2325 SExtPromotedInteger(
N->getOperand(0))), 0);
2328SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2330 SExtPromotedInteger(
N->getOperand(1))), 0);
2338 SDValue Val = GetPromotedInteger(
N->getValue());
2342 N->getMemoryVT(),
N->getMemOperand());
2353 Mask = PromoteTargetBoolean(Mask, DataVT);
2359 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2360 DataOp = GetPromotedInteger(DataOp);
2363 N->getOffset(), Mask,
N->getMemoryVT(),
2364 N->getMemOperand(),
N->getAddressingMode(),
2365 true,
N->isCompressingStore());
2370 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2371 EVT DataVT =
N->getValueType(0);
2372 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2374 NewOps[OpNo] =
Mask;
2391 EVT DataVT =
N->getValueType(0);
2392 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2393 }
else if (OpNo == 4) {
2395 if (
N->isIndexSigned())
2397 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2399 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2401 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2415 bool TruncateStore =
N->isTruncatingStore();
2420 EVT DataVT =
N->getValue().getValueType();
2421 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2422 }
else if (OpNo == 4) {
2424 if (
N->isIndexSigned())
2426 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2428 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2430 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2431 TruncateStore =
true;
2435 SDLoc(
N), NewOps,
N->getMemOperand(),
2436 N->getIndexType(), TruncateStore);
2440 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2441 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2443 N->getOperand(1),
N->getOperand(2));
2448 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2450 ZExtPromotedInteger(
N->getOperand(0)),
2451 N->getOperand(1),
N->getOperand(2)),
2454 ZExtPromotedInteger(
N->getOperand(0))), 0);
2457SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2459 ZExtPromotedInteger(
N->getOperand(1))), 0);
2464 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2469SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2471 EVT VT =
N->getValueType(0);
2472 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2474 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2477 N->getOperand(0).getValueType());
2481 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2486SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2488 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2493 bool IsStrict =
N->isStrictFPOpcode();
2505 SDValue Op = SExtPromotedInteger(
N->getOperand(1));
2515 unsigned OpOffset = IsStrict ? 1 : 0;
2518 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2519 "POWI exponent should match with sizeof(int) when doing the libcall.");
2522 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2523 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2524 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2525 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2527 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2532 switch (
N->getOpcode()) {
2540 case ISD::VP_REDUCE_ADD:
2541 case ISD::VP_REDUCE_MUL:
2542 case ISD::VP_REDUCE_AND:
2543 case ISD::VP_REDUCE_OR:
2544 case ISD::VP_REDUCE_XOR:
2548 case ISD::VP_REDUCE_SMAX:
2549 case ISD::VP_REDUCE_SMIN:
2553 case ISD::VP_REDUCE_UMAX:
2554 case ISD::VP_REDUCE_UMIN:
2564 return GetPromotedInteger(V);
2566 return SExtPromotedInteger(V);
2568 return ZExtPromotedInteger(V);
2574 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2576 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2577 EVT InVT =
Op.getValueType();
2579 EVT ResVT =
N->getValueType(0);
2580 unsigned Opcode =
N->getOpcode();
2600 Op = ZExtPromotedInteger(
N->getOperand(0));
2603 Op = SExtPromotedInteger(
N->getOperand(0));
2619 Op = ZExtPromotedInteger(
N->getOperand(0));
2622 Op = SExtPromotedInteger(
N->getOperand(0));
2636SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2643 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2647 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2649 Op = PromoteIntOpVectorReduction(
N,
Op);
2653 EVT VT =
N->getValueType(0);
2654 EVT EltVT =
Op.getValueType().getScalarType();
2669 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2673SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2676 SDValue Operand =
N->getOperand(OpNo);
2682SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2685 SDValue Operand =
N->getOperand(OpNo);
2691SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2692 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2693 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2696 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2701SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2705 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2709 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2711 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2723void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2729 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2732 switch (
N->getOpcode()) {
2735 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2736 N->dump(&DAG);
dbgs() <<
"\n";
2782 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2809 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2810 SplitInteger(Tmp.first,
Lo,
Hi);
2811 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2819 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2828 SplitInteger(Tmp,
Lo,
Hi);
2907 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2912 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2916 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2926std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2927 unsigned Opc =
Node->getOpcode();
2933 EVT RetVT =
Node->getValueType(0);
2941 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2942 "Unexpected atomic op or value type!");
2946 Node->getOperand(0));
2951void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2956 GetExpandedInteger(
N->getOperand(0), InL, InH);
2967 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2971 if (Amt.
uge(VTBits)) {
2973 }
else if (Amt.
ugt(NVTBits)) {
2977 }
else if (Amt == NVTBits) {
2994 if (Amt.
uge(VTBits)) {
2996 }
else if (Amt.
ugt(NVTBits)) {
3000 }
else if (Amt == NVTBits) {
3017 if (Amt.
uge(VTBits)) {
3020 }
else if (Amt.
ugt(NVTBits)) {
3025 }
else if (Amt == NVTBits) {
3045bool DAGTypeLegalizer::
3047 unsigned Opc =
N->getOpcode();
3055 "Expanded integer type size not a power of two!");
3062 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
3067 GetExpandedInteger(In, InL, InH);
3121 Lo = DAG.
getNode(Opc, dl, NVT, InL, Amt);
3134bool DAGTypeLegalizer::
3141 "Expanded integer type size not a power of two!");
3146 GetExpandedInteger(
N->getOperand(0), InL, InH);
3158 switch (
N->getOpcode()) {
3173 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3189 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3205 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3226void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3235 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3236 unsigned NumHalfBits = NumBits / 2;
3239 SDValue LHSL, LHSH, RHSL, RHSH;
3240 GetExpandedInteger(LHS, LHSL, LHSH);
3241 GetExpandedInteger(RHS, RHSL, RHSH);
3254 SDValue LHSL, LHSH, RHSL, RHSH;
3255 GetExpandedInteger(LHS, LHSL, LHSH);
3256 GetExpandedInteger(RHS, RHSL, RHSH);
3258 EVT CCT = getSetCCResultType(NVT);
3267 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3271 const APInt *RHSVal =
nullptr;
3272 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3273 RHSVal = &RHSConst->getAPIntValue();
3280 SDValue LHSL, LHSH, RHSL, RHSH;
3281 GetExpandedInteger(LHS, LHSL, LHSH);
3282 GetExpandedInteger(RHS, RHSL, RHSH);
3284 EVT CCT = getSetCCResultType(NVT);
3290 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3309 switch (
N->getOpcode()) {
3336 EVT VT =
N->getValueType(0);
3337 EVT CCT = getSetCCResultType(VT);
3340 SplitInteger(Result,
Lo,
Hi);
3345 SplitInteger(ExpandedCMP,
Lo,
Hi);
3348void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3352 SDValue LHSL, LHSH, RHSL, RHSH;
3353 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3354 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3357 SDValue LoOps[2] = { LHSL, RHSL };
3358 SDValue HiOps[3] = { LHSH, RHSH };
3367 HiOps[2] =
Lo.getValue(1);
3373 HiOps[2] =
Lo.getValue(1);
3395 HiOps[2] =
Lo.getValue(1);
3399 HiOps[2] =
Lo.getValue(1);
3412 EVT OvfVT = getSetCCResultType(NVT);
3452 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3455 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3458 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3476 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3490void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3493 SDValue LHSL, LHSH, RHSL, RHSH;
3495 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3496 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3498 SDValue LoOps[2] = { LHSL, RHSL };
3499 SDValue HiOps[3] = { LHSH, RHSH };
3503 HiOps[2] =
Lo.getValue(1);
3507 HiOps[2] =
Lo.getValue(1);
3513 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3516void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3519 SDValue LHSL, LHSH, RHSL, RHSH;
3521 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3522 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3524 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3525 SDValue HiOps[3] = { LHSH, RHSH };
3527 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3528 HiOps[2] =
Lo.getValue(1);
3529 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3533 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3536void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3544 unsigned CarryOp, NoCarryOp;
3546 switch(
N->getOpcode()) {
3566 SDValue LHSL, LHSH, RHSL, RHSH;
3567 GetExpandedInteger(LHS, LHSL, LHSH);
3568 GetExpandedInteger(RHS, RHSL, RHSH);
3570 SDValue LoOps[2] = { LHSL, RHSL };
3571 SDValue HiOps[3] = { LHSH, RHSH };
3573 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3574 HiOps[2] =
Lo.getValue(1);
3575 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3577 Ovf =
Hi.getValue(1);
3582 SplitInteger(Sum,
Lo,
Hi);
3593 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3604 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3610 SDValue LHSL, LHSH, RHSL, RHSH;
3612 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3613 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3615 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3618 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3619 HiOps[2] =
Lo.getValue(1);
3620 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3624 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3627void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3630 SDValue LHSL, LHSH, RHSL, RHSH;
3632 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3633 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3639 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3640 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3644 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3647void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3652 if (
Op.getValueType().bitsLE(NVT)) {
3659 assert(getTypeAction(
Op.getValueType()) ==
3661 "Only know how to promote this result!");
3664 "Operand over promoted?");
3666 SplitInteger(Res,
Lo,
Hi);
3670void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3673 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3674 EVT NVT =
Lo.getValueType();
3675 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3679 if (NVTBits < EVTBits) {
3682 EVTBits - NVTBits)));
3692void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3695 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3696 EVT NVT =
Lo.getValueType();
3697 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3701 if (NVTBits < EVTBits) {
3704 EVTBits - NVTBits)));
3712void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3715 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3720void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3723 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3732 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3733 EVT NVT =
Lo.getValueType();
3739void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3743 auto Constant = cast<ConstantSDNode>(
N);
3745 bool IsTarget =
Constant->isTargetOpcode();
3746 bool IsOpaque =
Constant->isOpaque();
3757 GetExpandedInteger(N0,
Lo,
Hi);
3758 EVT NVT =
Lo.getValueType();
3789 EVT VT =
N->getValueType(0);
3793 SplitInteger(Neg, NegLo, NegHi);
3801void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3805 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3806 EVT NVT =
Lo.getValueType();
3821void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3825 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3826 EVT NVT =
Lo.getValueType();
3832void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3836 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3837 EVT NVT =
Lo.getValueType();
3866 ReplaceValueWith(
SDValue(
N, 1), Chain);
3874 Chain =
Op.getValue(1);
3883 EVT VT =
N->getValueType(0);
3887 bool IsStrict =
N->isStrictFPOpcode();
3889 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3891 Op = GetPromotedFloat(
Op);
3894 EVT OFPVT =
Op.getValueType();
3896 Op = GetSoftPromotedHalf(
Op);
3900 SplitInteger(
Op,
Lo,
Hi);
3904 if (
Op.getValueType() == MVT::bf16) {
3911 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
3914 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3915 CallOptions, dl, Chain);
3916 SplitInteger(Tmp.first,
Lo,
Hi);
3919 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3925 SplitInteger(Res,
Lo,
Hi);
3931 bool IsStrict =
N->isStrictFPOpcode();
3932 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3936 "Input type needs to be promoted!");
3938 EVT VT =
Op.getValueType();
3940 if (VT == MVT::f16) {
3950 LC = RTLIB::LROUND_F32;
3951 else if (VT == MVT::f64)
3952 LC = RTLIB::LROUND_F64;
3953 else if (VT == MVT::f80)
3954 LC = RTLIB::LROUND_F80;
3955 else if (VT == MVT::f128)
3956 LC = RTLIB::LROUND_F128;
3957 else if (VT == MVT::ppcf128)
3958 LC = RTLIB::LROUND_PPCF128;
3959 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3963 LC = RTLIB::LRINT_F32;
3964 else if (VT == MVT::f64)
3965 LC = RTLIB::LRINT_F64;
3966 else if (VT == MVT::f80)
3967 LC = RTLIB::LRINT_F80;
3968 else if (VT == MVT::f128)
3969 LC = RTLIB::LRINT_F128;
3970 else if (VT == MVT::ppcf128)
3971 LC = RTLIB::LRINT_PPCF128;
3972 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
3976 LC = RTLIB::LLROUND_F32;
3977 else if (VT == MVT::f64)
3978 LC = RTLIB::LLROUND_F64;
3979 else if (VT == MVT::f80)
3980 LC = RTLIB::LLROUND_F80;
3981 else if (VT == MVT::f128)
3982 LC = RTLIB::LLROUND_F128;
3983 else if (VT == MVT::ppcf128)
3984 LC = RTLIB::LLROUND_PPCF128;
3985 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3989 LC = RTLIB::LLRINT_F32;
3990 else if (VT == MVT::f64)
3991 LC = RTLIB::LLRINT_F64;
3992 else if (VT == MVT::f80)
3993 LC = RTLIB::LLRINT_F80;
3994 else if (VT == MVT::f128)
3995 LC = RTLIB::LLRINT_F128;
3996 else if (VT == MVT::ppcf128)
3997 LC = RTLIB::LLRINT_PPCF128;
3998 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
4002 EVT RetVT =
N->getValueType(0);
4006 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
4007 Op, CallOptions, dl,
4009 SplitInteger(Tmp.first,
Lo,
Hi);
4011 if (
N->isStrictFPOpcode())
4012 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
4015void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
4017 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
4020 ExpandRes_NormalLoad(
N,
Lo,
Hi);
4026 EVT VT =
N->getValueType(0);
4037 if (
N->getMemoryVT().bitsLE(NVT)) {
4038 EVT MemVT =
N->getMemoryVT();
4041 N->getOriginalAlign(), MMOFlags, AAInfo);
4044 Ch =
Lo.getValue(1);
4049 unsigned LoSize =
Lo.getValueSizeInBits();
4064 N->getOriginalAlign(), MMOFlags, AAInfo);
4066 unsigned ExcessBits =
4074 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4075 N->getOriginalAlign(), MMOFlags, AAInfo);
4084 EVT MemVT =
N->getMemoryVT();
4087 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4093 N->getOriginalAlign(), MMOFlags, AAInfo);
4099 N->getPointerInfo().getWithOffset(IncrementSize),
4101 N->getOriginalAlign(), MMOFlags, AAInfo);
4125 ReplaceValueWith(
SDValue(
N, 1), Ch);
4128void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4132 GetExpandedInteger(
N->getOperand(0), LL, LH);
4133 GetExpandedInteger(
N->getOperand(1), RL, RH);
4138void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4140 EVT VT =
N->getValueType(0);
4145 GetExpandedInteger(
N->getOperand(0), LL, LH);
4146 GetExpandedInteger(
N->getOperand(1), RL, RH);
4156 LC = RTLIB::MUL_I16;
4157 else if (VT == MVT::i32)
4158 LC = RTLIB::MUL_I32;
4159 else if (VT == MVT::i64)
4160 LC = RTLIB::MUL_I64;
4161 else if (VT == MVT::i128)
4162 LC = RTLIB::MUL_I128;
4174 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4177 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4189 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4194 SplitInteger(Result,
Lo,
Hi);
4200 SplitInteger(Result,
Lo,
Hi);
4206 SplitInteger(Result,
Lo,
Hi);
4217 EVT VT =
N->getValueType(0);
4221 uint64_t Scale =
N->getConstantOperandVal(2);
4233 EVT BoolVT = getSetCCResultType(VT);
4258 SplitInteger(Result,
Lo,
Hi);
4264 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4268 GetExpandedInteger(LHS, LL, LH);
4269 GetExpandedInteger(RHS, RL, RH);
4273 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4281 SplitInteger(LoTmp, Result[0], Result[1]);
4282 SplitInteger(HiTmp, Result[2], Result[3]);
4284 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4287 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4288 "the size of the current value type");
4310 if (Scale % NVTSize) {
4326 if (Scale == VTSize)
4347 EVT BoolNVT = getSetCCResultType(NVT);
4350 if (Scale < NVTSize) {
4357 }
else if (Scale == NVTSize) {
4360 }
else if (Scale < VTSize) {
4368 "(and saturation can't happen with Scale==VTSize).");
4375 if (Scale < NVTSize) {
4380 unsigned OverflowBits = VTSize - Scale + 1;
4381 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4382 "Extent of overflow bits must start within HL");
4399 }
else if (Scale == NVTSize) {
4412 }
else if (Scale < VTSize) {
4415 unsigned OverflowBits = VTSize - Scale + 1;
4442 N->getConstantOperandVal(2), DAG);
4446 N->getConstantOperandVal(2), TLI, DAG);
4447 SplitInteger(Res,
Lo,
Hi);
4450void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4453 "Node has unexpected Opcode");
4468 SDValue LHSL, LHSH, RHSL, RHSH;
4469 GetExpandedInteger(LHS, LHSL, LHSH);
4470 GetExpandedInteger(RHS, RHSL, RHSH);
4474 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4476 Ovf =
Hi.getValue(1);
4483 SplitInteger(Sum,
Lo,
Hi);
4508 EVT VT =
LHS.getValueType();
4511 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4515 EVT OType =
Node->getValueType(1);
4523void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4525 EVT VT =
N->getValueType(0);
4527 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4537 LC = RTLIB::SDIV_I16;
4538 else if (VT == MVT::i32)
4539 LC = RTLIB::SDIV_I32;
4540 else if (VT == MVT::i64)
4541 LC = RTLIB::SDIV_I64;
4542 else if (VT == MVT::i128)
4543 LC = RTLIB::SDIV_I128;
4544 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4548 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4551void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4554 SDValue Shiftee =
N->getOperand(0);
4561 bool ShiftByByteMultiple =
4566 if (!ShiftByByteMultiple)
4570 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4571 unsigned VTByteWidth = VTBitWidth / 8;
4573 "Shiftee type size is not a power of two!");
4574 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4575 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4581 Align StackSlotAlignment(1);
4589 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4594 unsigned WideningOpc =
4596 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4603 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4609 if (ShiftByByteMultiple)
4610 Flags.setExact(
true);
4622 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4624 WillIndexUpwards = !WillIndexUpwards;
4627 if (WillIndexUpwards) {
4631 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4632 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4641 VT, dl, Ch, AdjStackPtr,
4646 if (!ShiftByByteMultiple) {
4649 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4653 SplitInteger(Res,
Lo,
Hi);
4656void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4658 EVT VT =
N->getValueType(0);
4659 unsigned Opc =
N->getOpcode();
4665 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4669 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4688 const bool LegalOrCustom =
4692 unsigned ExpansionFactor = 1;
4694 for (
EVT TmpVT = NVT;;) {
4696 if (NewTMPVT == TmpVT)
4706 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
4708 if (LegalOrCustom &&
4712 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4718 SDValue ShiftOp =
N->getOperand(1);
4723 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4725 Hi =
Lo.getValue(1);
4735 LC = RTLIB::SHL_I16;
4736 else if (VT == MVT::i32)
4737 LC = RTLIB::SHL_I32;
4738 else if (VT == MVT::i64)
4739 LC = RTLIB::SHL_I64;
4740 else if (VT == MVT::i128)
4741 LC = RTLIB::SHL_I128;
4745 LC = RTLIB::SRL_I16;
4746 else if (VT == MVT::i32)
4747 LC = RTLIB::SRL_I32;
4748 else if (VT == MVT::i64)
4749 LC = RTLIB::SRL_I64;
4750 else if (VT == MVT::i128)
4751 LC = RTLIB::SRL_I128;
4756 LC = RTLIB::SRA_I16;
4757 else if (VT == MVT::i32)
4758 LC = RTLIB::SRA_I32;
4759 else if (VT == MVT::i64)
4760 LC = RTLIB::SRA_I64;
4761 else if (VT == MVT::i128)
4762 LC = RTLIB::SRA_I128;
4769 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4772 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4776 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
4780void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4785 if (
Op.getValueType().bitsLE(NVT)) {
4796 assert(getTypeAction(
Op.getValueType()) ==
4798 "Only know how to promote this result!");
4801 "Operand over promoted?");
4803 SplitInteger(Res,
Lo,
Hi);
4811void DAGTypeLegalizer::
4814 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4815 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4837void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4839 EVT VT =
N->getValueType(0);
4841 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4851 LC = RTLIB::SREM_I16;
4852 else if (VT == MVT::i32)
4853 LC = RTLIB::SREM_I32;
4854 else if (VT == MVT::i64)
4855 LC = RTLIB::SREM_I64;
4856 else if (VT == MVT::i128)
4857 LC = RTLIB::SREM_I128;
4858 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4862 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4865void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4877void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
4879 EVT VT =
N->getValueType(0);
4898 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4899 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4900 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4902 EVT BitVT =
N->getValueType(1);
4927 SplitInteger(Three,
Lo,
Hi);
4931 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4942 LC = RTLIB::MULO_I32;
4943 else if (VT == MVT::i64)
4944 LC = RTLIB::MULO_I64;
4945 else if (VT == MVT::i128)
4946 LC = RTLIB::MULO_I128;
4959 SplitInteger(
Mul, MulLo, MulHi);
4965 SplitInteger(MulLo,
Lo,
Hi);
4966 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4979 EVT ArgVT =
Op.getValueType();
4983 Entry.IsSExt =
true;
4984 Entry.IsZExt =
false;
4985 Args.push_back(Entry);
4991 Entry.IsSExt =
true;
4992 Entry.IsZExt =
false;
4993 Args.push_back(Entry);
5012 ReplaceValueWith(
SDValue(
N, 1), Ofl);
5015void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
5017 EVT VT =
N->getValueType(0);
5019 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5028 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5031 if (isTypeLegal(NVT)) {
5033 GetExpandedInteger(
N->getOperand(0), InL, InH);
5045 LC = RTLIB::UDIV_I16;
5046 else if (VT == MVT::i32)
5047 LC = RTLIB::UDIV_I32;
5048 else if (VT == MVT::i64)
5049 LC = RTLIB::UDIV_I64;
5050 else if (VT == MVT::i128)
5051 LC = RTLIB::UDIV_I128;
5052 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
5055 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5058void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
5060 EVT VT =
N->getValueType(0);
5062 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5071 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5074 if (isTypeLegal(NVT)) {
5076 GetExpandedInteger(
N->getOperand(0), InL, InH);
5088 LC = RTLIB::UREM_I16;
5089 else if (VT == MVT::i32)
5090 LC = RTLIB::UREM_I32;
5091 else if (VT == MVT::i64)
5092 LC = RTLIB::UREM_I64;
5093 else if (VT == MVT::i128)
5094 LC = RTLIB::UREM_I128;
5095 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
5098 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5101void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
5106 if (
Op.getValueType().bitsLE(NVT)) {
5113 assert(getTypeAction(
Op.getValueType()) ==
5115 "Only know how to promote this result!");
5118 "Operand over promoted?");
5120 SplitInteger(Res,
Lo,
Hi);
5128void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5131 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5136 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5137 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5143void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5148 SplitInteger(Res,
Lo,
Hi);
5151void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5157 N->getOperand(0),
N->getOperand(1));
5158 SplitInteger(Res,
Lo,
Hi);
5165 GetExpandedInteger(
N->getOperand(0), In3, In4);
5166 GetExpandedInteger(
N->getOperand(1), In1, In2);
5170 unsigned Opc =
N->getOpcode();
5173 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5190 Lo = DAG.
getNode(Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5191 Hi = DAG.
getNode(Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5196 EVT VT =
N->getValueType(0);
5206 SplitInteger(Res,
Lo,
Hi);
5217bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5221 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5224 switch (
N->getOpcode()) {
5227 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5228 N->dump(&DAG);
dbgs() <<
"\n";
5233 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5240 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5246 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5253 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5258 case ISD::UCMP: Res = ExpandIntOp_CMP(
N);
break;
5262 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5265 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5267 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5268 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5269 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5274 if (!Res.
getNode())
return false;
5282 "Invalid operand expansion");
5284 ReplaceValueWith(
SDValue(
N, 0), Res);
5290void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5294 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5295 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5296 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5316 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5317 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5349 RHSLo, LowCC,
false, DagCombineInfo, dl);
5356 RHSHi, CCCode,
false, DagCombineInfo, dl);
5369 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5371 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5380 if (LHSHi == RHSHi) {
5393 if (HasSETCCCARRY) {
5396 bool FlipOperands =
false;
5424 false, DagCombineInfo, dl);
5433 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5434 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5435 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5447 N->getOperand(4)), 0);
5451 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5452 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5453 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5464 N->getOperand(2),
N->getOperand(3),
5469 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5470 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5471 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5476 "Unexpected setcc expansion!");
5492 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5493 GetExpandedInteger(LHS, LHSLo, LHSHi);
5494 GetExpandedInteger(RHS, RHSLo, RHSHi);
5507 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5517 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5530 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5535 bool IsStrict =
N->isStrictFPOpcode();
5539 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5540 EVT DstVT =
N->getValueType(0);
5543 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5544 "Don't know how to expand this XINT_TO_FP!");
5547 std::pair<SDValue, SDValue> Tmp =
5553 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5554 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5559 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5562 return ExpandOp_NormalStore(
N, OpNo);
5565 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5567 EVT VT =
N->getOperand(1).getValueType();
5578 if (
N->getMemoryVT().bitsLE(NVT)) {
5579 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5581 N->getMemoryVT(),
N->getOriginalAlign(), MMOFlags,
5587 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5590 N->getOriginalAlign(), MMOFlags, AAInfo);
5592 unsigned ExcessBits =
5600 N->getPointerInfo().getWithOffset(IncrementSize),
5601 NEVT,
N->getOriginalAlign(), MMOFlags, AAInfo);
5607 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5609 EVT ExtVT =
N->getMemoryVT();
5612 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5630 N->getOriginalAlign(), MMOFlags, AAInfo);
5636 N->getPointerInfo().getWithOffset(IncrementSize),
5638 N->getOriginalAlign(), MMOFlags, AAInfo);
5644 GetExpandedInteger(
N->getOperand(0), InL, InH);
5653 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5654 cast<AtomicSDNode>(
N)->getMemOperand());
5658SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5659 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5660 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5664 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5669SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5672 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5673 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5679SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
5682 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5683 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5692SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
5694 EVT OutVT =
N->getValueType(0);
5696 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5700 SDValue BaseIdx =
N->getOperand(1);
5727 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5736 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5740 "Promoted operand has an element type greater than result");
5753 InOp0 = GetPromotedInteger(InOp0);
5761 for (
unsigned i = 0; i != OutNumElems; ++i) {
5766 N->getOperand(0),
Index);
5775SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
5776 EVT OutVT =
N->getValueType(0);
5778 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5790 Vec = GetPromotedInteger(Vec);
5796SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
5799 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5805SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
5807 EVT VT =
N->getValueType(0);
5812 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5813 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5820 EVT OutVT =
N->getValueType(0);
5822 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5823 unsigned NumElems =
N->getNumOperands();
5831 for (
unsigned i = 0; i != NumElems; ++i) {
5833 EVT OpVT =
Op.getValueType();
5838 if (OpVT.
bitsLT(NOutVTElem)) {
5844 ExtOpc = NOutExtOpc;
5857 assert(!
N->getOperand(0).getValueType().isVector() &&
5858 "Input must be a scalar");
5860 EVT OutVT =
N->getValueType(0);
5862 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5867 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
5872 EVT OutVT =
N->getValueType(0);
5875 "Type must be promoted to a scalable vector type");
5876 const APInt &StepVal =
N->getConstantOperandAPInt(0);
5881SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
5884 EVT OutVT =
N->getValueType(0);
5886 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5888 unsigned NumOperands =
N->getNumOperands();
5893 SDUse *MaxSizedValue = std::max_element(
5895 EVT AVT = A.getValueType().getVectorElementType();
5896 EVT BVT = B.getValueType().getVectorElementType();
5897 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5903 for (
unsigned I = 0;
I < NumOperands; ++
I) {
5905 EVT OpVT =
Op.getValueType();
5907 Op = GetPromotedInteger(
Op);
5910 "Unhandled legalization type");
5927 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
5928 assert(NumElem * NumOperands == NumOutElem &&
5929 "Unexpected number of elements");
5933 for (
unsigned i = 0; i < NumOperands; ++i) {
5936 Op = GetPromotedInteger(
Op);
5937 EVT SclrTy =
Op.getValueType().getVectorElementType();
5938 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
5939 "Unexpected number of elements");
5941 for (
unsigned j = 0;
j < NumElem; ++
j) {
5951SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
5952 EVT VT =
N->getValueType(0);
5954 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
5962 if (getTypeAction(
N->getOperand(0).getValueType())
5966 switch(
N->getOpcode()) {
5968 Promoted = SExtPromotedInteger(
N->getOperand(0));
5971 Promoted = ZExtPromotedInteger(
N->getOperand(0));
5974 Promoted = GetPromotedInteger(
N->getOperand(0));
5979 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
5983 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
5986SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
5987 EVT OutVT =
N->getValueType(0);
5989 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5994 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5997 NOutVTElem,
N->getOperand(1));
5999 V0, ConvElem,
N->getOperand(2));
6007 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
6015 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
6016 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
6017 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
6024 assert(
N->getNumValues() == 3 &&
"Expected 3 values for PATCHPOINT");
6038SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
6040 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6052SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
6057 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6067SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
6069 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6072 N->getValueType(0).getVectorNumElements());
6077SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
6080 EVT ResVT =
N->getValueType(0);
6081 unsigned NumElems =
N->getNumOperands();
6086 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
6088 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
6096 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
6102 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
6104 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
6105 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
6107 for (
unsigned i=0; i<NumElem; ++i) {
6119SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
6132 for (
unsigned I = 0;
I < OpNo;
I++)
6135 EVT Ty =
Op.getValueType();
6147 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6152 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6158SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6171 for (
unsigned I = 0;
I < OpNo;
I++)
6174 EVT Ty =
Op.getValueType();
6186 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6191 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)