31#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::VP_BITREVERSE:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
79 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
80 case ISD::VP_CTTZ_ELTS:
81 Res = PromoteIntRes_VP_CttzElements(
N);
84 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
85 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
86 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
88 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
94 Res = PromoteIntRes_Select(
N);
99 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
101 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
103 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
106 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
108 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
110 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(
N);
break;
112 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(
N);
break;
113 case ISD::VP_TRUNCATE:
115 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
116 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
120 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
122 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
124 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
126 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
128 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
131 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
134 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
136 Res = PromoteIntRes_BUILD_VECTOR(
N);
140 Res = PromoteIntRes_ScalarOp(
N);
144 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
149 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
152 case ISD::VP_SIGN_EXTEND:
154 case ISD::VP_ZERO_EXTEND:
157 case ISD::VP_FP_TO_SINT:
158 case ISD::VP_FP_TO_UINT:
166 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
170 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
174 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
189 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
196 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
203 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
206 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
208 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
210 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
226 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
228 case ISD::VP_SADDSAT:
229 case ISD::VP_UADDSAT:
230 case ISD::VP_SSUBSAT:
231 case ISD::VP_USUBSAT:
232 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
245 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
248 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
262 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
266 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
278 Res = PromoteIntRes_VECREDUCE(
N);
281 case ISD::VP_REDUCE_ADD:
282 case ISD::VP_REDUCE_MUL:
283 case ISD::VP_REDUCE_AND:
284 case ISD::VP_REDUCE_OR:
285 case ISD::VP_REDUCE_XOR:
286 case ISD::VP_REDUCE_SMAX:
287 case ISD::VP_REDUCE_SMIN:
288 case ISD::VP_REDUCE_UMAX:
289 case ISD::VP_REDUCE_UMIN:
290 Res = PromoteIntRes_VP_REDUCE(
N);
294 Res = PromoteIntRes_FREEZE(
N);
299 Res = PromoteIntRes_Rotate(
N);
304 Res = PromoteIntRes_FunnelShift(
N);
309 Res = PromoteIntRes_VPFunnelShift(
N);
313 Res = PromoteIntRes_IS_FPCLASS(
N);
316 Res = PromoteIntRes_FFREXP(
N);
321 Res = PromoteIntRes_XRINT(
N);
327 SetPromotedInteger(
SDValue(
N, ResNo), Res);
332 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
333 return GetPromotedInteger(
Op);
338 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
340 Op.getValueType(),
Op,
N->getOperand(1));
345 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
347 Op.getValueType(),
Op,
N->getOperand(1));
353 N->getMemoryVT(), ResVT,
354 N->getChain(),
N->getBasePtr(),
373 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
383 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
386 N->getChain(),
N->getBasePtr(),
387 Op2,
N->getMemOperand());
398 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
409 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
419 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
422 Op2 = SExtPromotedInteger(Op2);
425 Op2 = ZExtPromotedInteger(Op2);
428 Op2 = GetPromotedInteger(Op2);
437 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
438 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
440 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
449 EVT OutVT =
N->getValueType(0);
453 switch (getTypeAction(InVT)) {
480 BitConvertToInteger(GetScalarizedVector(InOp)));
489 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
490 Lo = BitConvertToInteger(
Lo);
491 Hi = BitConvertToInteger(
Hi);
499 JoinIntegers(
Lo,
Hi));
533 if (isTypeLegal(WideOutVT)) {
534 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
544 CreateStackStoreLoad(InOp, OutVT));
548 SDValue V = GetPromotedInteger(
N->getOperand(0));
550 V.getValueType(), V);
554 SDValue Op = GetPromotedInteger(
N->getOperand(0));
555 EVT OVT =
N->getValueType(0);
556 EVT NVT =
Op.getValueType();
576 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
577 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
582 SDValue Op = GetPromotedInteger(
N->getOperand(0));
583 EVT OVT =
N->getValueType(0);
584 EVT NVT =
Op.getValueType();
604 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
605 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
614 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
619 EVT VT =
N->getValueType(0);
628 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
633 EVT OVT =
N->getValueType(0);
650 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
655 if (!
N->isVPOpcode())
661 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
662 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
663 ExtractLeadingBits, Mask, EVL);
667 EVT OVT =
N->getValueType(0);
684 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
685 if (!
N->isVPOpcode())
688 N->getOperand(1),
N->getOperand(2));
692 SDValue Op = GetPromotedInteger(
N->getOperand(0));
693 EVT OVT =
N->getValueType(0);
694 EVT NVT =
Op.getValueType();
712 if (
N->getOpcode() ==
ISD::CTTZ ||
N->getOpcode() == ISD::VP_CTTZ) {
723 N->getOperand(1),
N->getOperand(2));
725 if (!
N->isVPOpcode())
726 return DAG.
getNode(
N->getOpcode(), dl, NVT,
Op);
727 return DAG.
getNode(
N->getOpcode(), dl, NVT,
Op,
N->getOperand(1),
731SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(
SDNode *
N) {
734 return DAG.
getNode(
N->getOpcode(),
DL, NewVT,
N->ops());
737SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
752 EVT SVT =
In.getValueType().getScalarType();
764 unsigned NewOpc =
N->getOpcode();
781 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
784 NewOpc = ISD::VP_FP_TO_SINT;
787 if (
N->isStrictFPOpcode()) {
788 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
789 {
N->getOperand(0),
N->getOperand(1)});
793 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
794 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
797 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
809 N->getOpcode() == ISD::VP_FP_TO_UINT)
816SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
820 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
824SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
828 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
831SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
836 N->getOperand(0),
N->getOperand(1));
844 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
852 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
864 if (getTypeAction(
N->getOperand(0).getValueType())
866 SDValue Res = GetPromotedInteger(
N->getOperand(0));
884 if (
N->getNumOperands() != 1) {
885 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
886 assert(
N->isVPOpcode() &&
"Expected VP opcode");
887 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
888 N->getOperand(1),
N->getOperand(2));
890 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
900 N->getMemoryVT(),
N->getMemOperand());
910 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
918 N->getOffset(),
N->getMask(), ExtPassThru,
919 N->getMemoryVT(),
N->getMemOperand(),
920 N->getAddressingMode(), ExtType,
921 N->isExpandingLoad());
930 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
932 "Gather result type and the passThru argument type should be the same");
939 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
940 N->getIndex(),
N->getScale() };
942 N->getMemoryVT(), dl, Ops,
943 N->getMemOperand(),
N->getIndexType(),
956 EVT VT =
N->getValueType(0);
957 EVT SVT = getSetCCResultType(VT);
958 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
959 unsigned NumOps =
N->getNumOperands();
960 assert(NumOps <= 3 &&
"Too many operands");
962 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
970 ReplaceValueWith(
SDValue(
N, 0), Res);
976template <
class MatchContextClass>
988 MatchContextClass matcher(DAG, TLI,
N);
991 unsigned Opcode = matcher.getRootBaseOpcode();
995 SDValue Op1Promoted, Op2Promoted;
997 Op1Promoted = GetPromotedInteger(Op1);
998 Op2Promoted = ZExtPromotedInteger(Op2);
1000 Op1Promoted = ZExtPromotedInteger(Op1);
1001 Op2Promoted = ZExtPromotedInteger(Op2);
1003 Op1Promoted = SExtPromotedInteger(Op1);
1004 Op2Promoted = SExtPromotedInteger(Op2);
1013 matcher.getNode(
ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1014 return matcher.getNode(
ISD::UMIN, dl, PromotedType,
Add, SatMax);
1019 return matcher.getNode(
ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1024 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1037 "addition, subtraction or left shift");
1040 unsigned SHLAmount = NewBits - OldBits;
1047 matcher.
getNode(
ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1050 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1051 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1060 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1061 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1062 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1069 SDValue Op1Promoted, Op2Promoted;
1075 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1076 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1078 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1079 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1081 EVT OldType =
N->getOperand(0).getValueType();
1095 Op2Promoted,
N->getOperand(2));
1097 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1100 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1105 unsigned SatW,
bool Signed,
1108 EVT VT = V.getValueType();
1135 EVT VT =
LHS.getValueType();
1153 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1159 "Tried to saturate to more than the original type?");
1168 SDValue Op1Promoted, Op2Promoted;
1174 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1175 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1177 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1178 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1181 unsigned Scale =
N->getConstantOperandVal(2);
1190 N->getValueType(0).getScalarSizeInBits();
1195 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1196 Op2Promoted,
N->getOperand(2));
1206 Op2Promoted, Scale, DAG)) {
1209 N->getValueType(0).getScalarSizeInBits(),
1217 N->getValueType(0).getScalarSizeInBits());
1220SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1222 return PromoteIntRes_Overflow(
N);
1226 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1227 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1228 EVT OVT =
N->getOperand(0).getValueType();
1229 EVT NVT =
LHS.getValueType();
1244 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1252 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1253 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1255 unsigned Opcode =
N->getOpcode();
1256 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1263 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1264 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1266 LHS.getValueType(),
N->getOperand(0),
1267 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1271 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1272 EVT InVT =
N->getOperand(OpNo).getValueType();
1275 EVT SVT = getSetCCResultType(InVT);
1283 SVT = getSetCCResultType(InVT);
1291 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1292 "Vector compare must return a vector result!");
1296 if (
N->isStrictFPOpcode()) {
1298 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1299 N->getOperand(2),
N->getOperand(3)};
1300 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1305 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1306 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1322 EVT VT =
N->getValueType(0);
1328 ReplaceValueWith(
SDValue(
N, 0), Res);
1333 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1336 RHS = ZExtPromotedInteger(RHS);
1337 if (
N->getOpcode() != ISD::VP_SHL)
1340 N->getOperand(2),
N->getOperand(3));
1343SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1344 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1346 Op.getValueType(),
Op,
N->getOperand(1));
1349SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1353 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1354 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1355 if (
N->getNumOperands() == 2)
1357 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1358 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1360 N->getOperand(2),
N->getOperand(3));
1365 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1366 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1367 if (
N->getNumOperands() == 2)
1369 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1370 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1372 N->getOperand(2),
N->getOperand(3));
1377 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1378 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1379 if (
N->getNumOperands() == 2)
1381 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1382 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1384 N->getOperand(2),
N->getOperand(3));
1393 SExtOrZExtPromotedOperands(LHS, RHS);
1396 LHS.getValueType(), LHS, RHS);
1401 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1404 RHS = ZExtPromotedInteger(RHS);
1405 if (
N->getOpcode() != ISD::VP_ASHR)
1408 N->getOperand(2),
N->getOperand(3));
1413 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1416 RHS = ZExtPromotedInteger(RHS);
1417 if (
N->getOpcode() != ISD::VP_LSHR)
1420 N->getOperand(2),
N->getOperand(3));
1426 ReplaceValueWith(
SDValue(
N, 0), Res);
1431 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1432 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1435 Amt = ZExtPromotedInteger(Amt);
1439 EVT OldVT =
N->getOperand(0).getValueType();
1440 EVT VT =
Lo.getValueType();
1441 unsigned Opcode =
N->getOpcode();
1455 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1480SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1481 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1482 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1487 Amt = ZExtPromotedInteger(Amt);
1491 EVT OldVT =
N->getOperand(0).getValueType();
1492 EVT VT =
Lo.getValueType();
1493 unsigned Opcode =
N->getOpcode();
1494 bool IsFSHR = Opcode == ISD::VP_FSHR;
1499 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1507 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1510 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1514 Res = DAG.
getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL,
DL, VT, Res, Amt,
1517 Res = DAG.
getNode(ISD::VP_LSHR,
DL, VT, Res, HiShift, Mask, EVL);
1523 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1528 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1546 Res = GetPromotedInteger(InOp);
1553 "Dst and Src must have the same number of elements");
1555 "Promoted vector type must be a power of two");
1558 GetSplitVector(InOp, EOp1, EOp2);
1566 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1567 "Expected VP_TRUNCATE opcode");
1568 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1569 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1570 std::tie(EVLLo, EVLHi) =
1571 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1572 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1573 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1580 SDValue WideInOp = GetWidenedVector(InOp);
1585 N->getValueType(0).getScalarType(), NumElem);
1600 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1601 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1606SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1608 return PromoteIntRes_Overflow(
N);
1612 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1613 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1614 EVT OVT =
N->getOperand(0).getValueType();
1615 EVT NVT =
LHS.getValueType();
1629 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1640 return PromoteIntRes_Overflow(
N);
1652 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1653 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1655 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1659 LHS, RHS,
N->getOperand(2));
1669 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1670 return PromoteIntRes_Overflow(
N);
1674 EVT OVT =
N->getValueType(0);
1687 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1691SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1694 return PromoteIntRes_Overflow(
N);
1698 EVT SmallVT =
LHS.getValueType();
1705 LHS = SExtPromotedInteger(LHS);
1706 RHS = SExtPromotedInteger(RHS);
1708 LHS = ZExtPromotedInteger(LHS);
1709 RHS = ZExtPromotedInteger(RHS);
1740 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1746 N->getValueType(0)));
1752 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1759 EVT VT =
N->getValueType(0);
1767 for (
unsigned i = 0; i < NumRegs; ++i) {
1768 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1769 N->getConstantOperandVal(3));
1775 std::reverse(Parts.begin(), Parts.end());
1780 for (
unsigned i = 1; i < NumRegs; ++i) {
1791 ReplaceValueWith(
SDValue(
N, 1), Chain);
1804bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1807 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1812 switch (
N->getOpcode()) {
1815 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1816 N->dump(&DAG);
dbgs() <<
"\n";
1822 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1825 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1826 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1832 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1836 Res = PromoteIntOp_ScalarOp(
N);
1839 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1842 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1844 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
1845 case ISD::VP_SINT_TO_FP:
1848 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1850 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1852 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1854 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1856 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1858 case ISD::VP_TRUNCATE:
1862 case ISD::VP_UINT_TO_FP:
1867 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
1875 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1878 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1904 case ISD::VP_REDUCE_ADD:
1905 case ISD::VP_REDUCE_MUL:
1906 case ISD::VP_REDUCE_AND:
1907 case ISD::VP_REDUCE_OR:
1908 case ISD::VP_REDUCE_XOR:
1909 case ISD::VP_REDUCE_SMAX:
1910 case ISD::VP_REDUCE_SMIN:
1911 case ISD::VP_REDUCE_UMAX:
1912 case ISD::VP_REDUCE_UMIN:
1913 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
1918 Res = PromoteIntOp_STACKMAP(
N, OpNo);
1921 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
1923 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1924 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1925 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
1927 case ISD::EXPERIMENTAL_VP_SPLICE:
1928 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
1933 if (!Res.
getNode())
return false;
1940 const bool IsStrictFp =
N->isStrictFPOpcode();
1942 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1943 "Invalid operand expansion");
1947 ReplaceValueWith(
SDValue(
N, 0), Res);
1957void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
1958 SDValue OpL = GetPromotedInteger(LHS);
1959 SDValue OpR = GetPromotedInteger(RHS);
1965 unsigned OpLEffectiveBits =
1967 unsigned OpREffectiveBits =
1969 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1970 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1977 LHS = SExtPromotedInteger(LHS);
1978 RHS = SExtPromotedInteger(RHS);
1989 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1990 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1997 LHS = ZExtPromotedInteger(LHS);
1998 RHS = ZExtPromotedInteger(RHS);
2003void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2012 LHS = SExtPromotedInteger(LHS);
2013 RHS = SExtPromotedInteger(RHS);
2018 "Unknown integer comparison!");
2020 SExtOrZExtPromotedOperands(LHS, RHS);
2024 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2029 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2031 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2037 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
2040SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2041 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2045 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2050 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2054SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2055 assert(OpNo == 1 &&
"only know how to promote condition");
2058 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2062 N->getOperand(2)), 0);
2067 EVT OVT =
N->getOperand(0).getValueType();
2068 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2069 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2070 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2083 EVT VecVT =
N->getValueType(0);
2086 "Legal vector of one illegal element?");
2091 assert(
N->getOperand(0).getValueSizeInBits() >=
2092 N->getValueType(0).getScalarSizeInBits() &&
2093 "Type of inserted value narrower than vector element type!");
2096 for (
unsigned i = 0; i < NumElts; ++i)
2097 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2102SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2109 assert(
N->getOperand(1).getValueSizeInBits() >=
2110 N->getValueType(0).getScalarSizeInBits() &&
2111 "Type of inserted value narrower than vector element type!");
2113 GetPromotedInteger(
N->getOperand(1)),
2118 assert(OpNo == 2 &&
"Different operand and result vector types?");
2124 N->getOperand(1),
Idx), 0);
2131 GetPromotedInteger(
N->getOperand(0))), 0);
2134SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2135 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2137 EVT OpTy =
N->getOperand(1).getValueType();
2140 if (
SDValue Res = WidenVSELECTMask(
N))
2142 Res,
N->getOperand(1),
N->getOperand(2));
2146 Cond = PromoteTargetBoolean(
Cond, OpVT);
2149 N->getOperand(2)), 0);
2152SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2153 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2157 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2161 N->getOperand(3),
N->getOperand(4)), 0);
2164SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2165 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2169 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2175 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2178 N->getOperand(3),
N->getOperand(4)),
2184 ZExtPromotedInteger(
N->getOperand(1))), 0);
2189 ZExtPromotedInteger(
N->getOperand(2))), 0);
2193 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2200SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2202 EVT VT =
N->getValueType(0);
2203 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2205 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2213 return DAG.
getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt,
N->getOperand(1),
2218 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2220 SExtPromotedInteger(
N->getOperand(0)),
2221 N->getOperand(1),
N->getOperand(2)),
2224 SExtPromotedInteger(
N->getOperand(0))), 0);
2227SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2229 SExtPromotedInteger(
N->getOperand(1))), 0);
2237 SDValue Val = GetPromotedInteger(
N->getValue());
2241 N->getMemoryVT(),
N->getMemOperand());
2252 Mask = PromoteTargetBoolean(Mask, DataVT);
2258 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2259 DataOp = GetPromotedInteger(DataOp);
2262 N->getOffset(), Mask,
N->getMemoryVT(),
2263 N->getMemOperand(),
N->getAddressingMode(),
2264 true,
N->isCompressingStore());
2269 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2270 EVT DataVT =
N->getValueType(0);
2271 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2273 NewOps[OpNo] =
Mask;
2290 EVT DataVT =
N->getValueType(0);
2291 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2292 }
else if (OpNo == 4) {
2294 if (
N->isIndexSigned())
2296 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2298 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2300 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2314 bool TruncateStore =
N->isTruncatingStore();
2319 EVT DataVT =
N->getValue().getValueType();
2320 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2321 }
else if (OpNo == 4) {
2323 if (
N->isIndexSigned())
2325 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2327 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2329 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2330 TruncateStore =
true;
2334 SDLoc(
N), NewOps,
N->getMemOperand(),
2335 N->getIndexType(), TruncateStore);
2339 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2340 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2342 N->getOperand(1),
N->getOperand(2));
2347 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2349 ZExtPromotedInteger(
N->getOperand(0)),
2350 N->getOperand(1),
N->getOperand(2)),
2353 ZExtPromotedInteger(
N->getOperand(0))), 0);
2356SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2358 ZExtPromotedInteger(
N->getOperand(1))), 0);
2363 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2368SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2370 EVT VT =
N->getValueType(0);
2371 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2373 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2376 N->getOperand(0).getScalarValueSizeInBits());
2378 N->getOperand(1),
N->getOperand(2));
2382 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2387SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2389 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2394 bool IsStrict =
N->isStrictFPOpcode();
2406 SDValue Op = SExtPromotedInteger(
N->getOperand(1));
2416 unsigned OpOffset = IsStrict ? 1 : 0;
2419 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2420 "POWI exponent should match with sizeof(int) when doing the libcall.");
2423 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2424 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2425 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2426 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2428 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2433 switch (
N->getOpcode()) {
2441 case ISD::VP_REDUCE_ADD:
2442 case ISD::VP_REDUCE_MUL:
2443 case ISD::VP_REDUCE_AND:
2444 case ISD::VP_REDUCE_OR:
2445 case ISD::VP_REDUCE_XOR:
2449 case ISD::VP_REDUCE_SMAX:
2450 case ISD::VP_REDUCE_SMIN:
2454 case ISD::VP_REDUCE_UMAX:
2455 case ISD::VP_REDUCE_UMIN:
2465 return GetPromotedInteger(V);
2467 return SExtPromotedInteger(V);
2469 return ZExtPromotedInteger(V);
2475 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2477 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2478 EVT InVT =
Op.getValueType();
2480 EVT ResVT =
N->getValueType(0);
2481 unsigned Opcode =
N->getOpcode();
2501 Op = ZExtPromotedInteger(
N->getOperand(0));
2504 Op = SExtPromotedInteger(
N->getOperand(0));
2520 Op = ZExtPromotedInteger(
N->getOperand(0));
2523 Op = SExtPromotedInteger(
N->getOperand(0));
2537SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2544 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2548 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2550 Op = PromoteIntOpVectorReduction(
N,
Op);
2554 EVT VT =
N->getValueType(0);
2555 EVT EltVT =
Op.getValueType().getScalarType();
2570 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2574SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2577 SDValue Operand =
N->getOperand(OpNo);
2583SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2586 SDValue Operand =
N->getOperand(OpNo);
2592SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2593 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2594 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2597 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2602SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2606 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2610 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2612 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2624void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2630 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2633 switch (
N->getOpcode()) {
2636 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2637 N->dump(&DAG);
dbgs() <<
"\n";
2683 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2710 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2711 SplitInteger(Tmp.first,
Lo,
Hi);
2712 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2720 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2729 SplitInteger(Tmp,
Lo,
Hi);
2800 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2805 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2809 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2819std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2820 unsigned Opc =
Node->getOpcode();
2826 EVT RetVT =
Node->getValueType(0);
2834 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2835 "Unexpected atomic op or value type!");
2839 Node->getOperand(0));
2844void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2849 GetExpandedInteger(
N->getOperand(0), InL, InH);
2860 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2864 if (Amt.
uge(VTBits)) {
2866 }
else if (Amt.
ugt(NVTBits)) {
2870 }
else if (Amt == NVTBits) {
2887 if (Amt.
uge(VTBits)) {
2889 }
else if (Amt.
ugt(NVTBits)) {
2893 }
else if (Amt == NVTBits) {
2910 if (Amt.
uge(VTBits)) {
2913 }
else if (Amt.
ugt(NVTBits)) {
2918 }
else if (Amt == NVTBits) {
2938bool DAGTypeLegalizer::
2940 unsigned Opc =
N->getOpcode();
2948 "Expanded integer type size not a power of two!");
2955 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
2960 GetExpandedInteger(In, InL, InH);
3014 Lo = DAG.
getNode(Opc, dl, NVT, InL, Amt);
3027bool DAGTypeLegalizer::
3034 "Expanded integer type size not a power of two!");
3039 GetExpandedInteger(
N->getOperand(0), InL, InH);
3051 switch (
N->getOpcode()) {
3066 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3082 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3098 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3119void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3128 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3129 unsigned NumHalfBits = NumBits / 2;
3132 SDValue LHSL, LHSH, RHSL, RHSH;
3133 GetExpandedInteger(LHS, LHSL, LHSH);
3134 GetExpandedInteger(RHS, RHSL, RHSH);
3147 SDValue LHSL, LHSH, RHSL, RHSH;
3148 GetExpandedInteger(LHS, LHSL, LHSH);
3149 GetExpandedInteger(RHS, RHSL, RHSH);
3151 EVT CCT = getSetCCResultType(NVT);
3160 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3164 const APInt *RHSVal =
nullptr;
3165 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3166 RHSVal = &RHSConst->getAPIntValue();
3173 SDValue LHSL, LHSH, RHSL, RHSH;
3174 GetExpandedInteger(LHS, LHSL, LHSH);
3175 GetExpandedInteger(RHS, RHSL, RHSH);
3177 EVT CCT = getSetCCResultType(NVT);
3183 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3202 switch (
N->getOpcode()) {
3229 EVT VT =
N->getValueType(0);
3230 EVT CCT = getSetCCResultType(VT);
3233 SplitInteger(Result,
Lo,
Hi);
3236void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3240 SDValue LHSL, LHSH, RHSL, RHSH;
3241 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3242 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3245 SDValue LoOps[2] = { LHSL, RHSL };
3246 SDValue HiOps[3] = { LHSH, RHSH };
3255 HiOps[2] =
Lo.getValue(1);
3261 HiOps[2] =
Lo.getValue(1);
3283 HiOps[2] =
Lo.getValue(1);
3287 HiOps[2] =
Lo.getValue(1);
3300 EVT OvfVT = getSetCCResultType(NVT);
3340 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3343 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3346 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3364 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3378void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3381 SDValue LHSL, LHSH, RHSL, RHSH;
3383 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3384 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3386 SDValue LoOps[2] = { LHSL, RHSL };
3387 SDValue HiOps[3] = { LHSH, RHSH };
3391 HiOps[2] =
Lo.getValue(1);
3395 HiOps[2] =
Lo.getValue(1);
3401 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3404void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3407 SDValue LHSL, LHSH, RHSL, RHSH;
3409 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3410 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3412 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3413 SDValue HiOps[3] = { LHSH, RHSH };
3415 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3416 HiOps[2] =
Lo.getValue(1);
3417 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3421 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3424void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3432 unsigned CarryOp, NoCarryOp;
3434 switch(
N->getOpcode()) {
3454 SDValue LHSL, LHSH, RHSL, RHSH;
3455 GetExpandedInteger(LHS, LHSL, LHSH);
3456 GetExpandedInteger(RHS, RHSL, RHSH);
3458 SDValue LoOps[2] = { LHSL, RHSL };
3459 SDValue HiOps[3] = { LHSH, RHSH };
3461 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3462 HiOps[2] =
Lo.getValue(1);
3463 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3465 Ovf =
Hi.getValue(1);
3470 SplitInteger(Sum,
Lo,
Hi);
3481 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3492 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3498 SDValue LHSL, LHSH, RHSL, RHSH;
3500 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3501 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3503 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3506 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3507 HiOps[2] =
Lo.getValue(1);
3508 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3512 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3515void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3518 SDValue LHSL, LHSH, RHSL, RHSH;
3520 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3521 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3527 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3528 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3532 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3535void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3540 if (
Op.getValueType().bitsLE(NVT)) {
3547 assert(getTypeAction(
Op.getValueType()) ==
3549 "Only know how to promote this result!");
3552 "Operand over promoted?");
3554 SplitInteger(Res,
Lo,
Hi);
3558void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3561 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3562 EVT NVT =
Lo.getValueType();
3563 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3567 if (NVTBits < EVTBits) {
3570 EVTBits - NVTBits)));
3580void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3583 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3584 EVT NVT =
Lo.getValueType();
3585 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3589 if (NVTBits < EVTBits) {
3592 EVTBits - NVTBits)));
3600void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3603 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3608void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3611 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3620 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3621 EVT NVT =
Lo.getValueType();
3627void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3631 auto Constant = cast<ConstantSDNode>(
N);
3633 bool IsTarget =
Constant->isTargetOpcode();
3634 bool IsOpaque =
Constant->isOpaque();
3645 GetExpandedInteger(N0,
Lo,
Hi);
3646 EVT NVT =
Lo.getValueType();
3677 EVT VT =
N->getValueType(0);
3681 SplitInteger(Neg, NegLo, NegHi);
3689void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3693 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3694 EVT NVT =
Lo.getValueType();
3709void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3713 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3714 EVT NVT =
Lo.getValueType();
3720void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3724 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3725 EVT NVT =
Lo.getValueType();
3754 ReplaceValueWith(
SDValue(
N, 1), Chain);
3762 Chain =
Op.getValue(1);
3771 EVT VT =
N->getValueType(0);
3775 bool IsStrict =
N->isStrictFPOpcode();
3777 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3779 Op = GetPromotedFloat(
Op);
3782 EVT OFPVT =
Op.getValueType();
3784 Op = GetSoftPromotedHalf(
Op);
3788 SplitInteger(
Op,
Lo,
Hi);
3792 if (
Op.getValueType() == MVT::bf16) {
3799 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
3802 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3803 CallOptions, dl, Chain);
3804 SplitInteger(Tmp.first,
Lo,
Hi);
3807 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3813 SplitInteger(Res,
Lo,
Hi);
3819 bool IsStrict =
N->isStrictFPOpcode();
3820 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3824 "Input type needs to be promoted!");
3826 EVT VT =
Op.getValueType();
3828 if (VT == MVT::f16) {
3838 LC = RTLIB::LROUND_F32;
3839 else if (VT == MVT::f64)
3840 LC = RTLIB::LROUND_F64;
3841 else if (VT == MVT::f80)
3842 LC = RTLIB::LROUND_F80;
3843 else if (VT == MVT::f128)
3844 LC = RTLIB::LROUND_F128;
3845 else if (VT == MVT::ppcf128)
3846 LC = RTLIB::LROUND_PPCF128;
3847 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3851 LC = RTLIB::LRINT_F32;
3852 else if (VT == MVT::f64)
3853 LC = RTLIB::LRINT_F64;
3854 else if (VT == MVT::f80)
3855 LC = RTLIB::LRINT_F80;
3856 else if (VT == MVT::f128)
3857 LC = RTLIB::LRINT_F128;
3858 else if (VT == MVT::ppcf128)
3859 LC = RTLIB::LRINT_PPCF128;
3860 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
3864 LC = RTLIB::LLROUND_F32;
3865 else if (VT == MVT::f64)
3866 LC = RTLIB::LLROUND_F64;
3867 else if (VT == MVT::f80)
3868 LC = RTLIB::LLROUND_F80;
3869 else if (VT == MVT::f128)
3870 LC = RTLIB::LLROUND_F128;
3871 else if (VT == MVT::ppcf128)
3872 LC = RTLIB::LLROUND_PPCF128;
3873 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3877 LC = RTLIB::LLRINT_F32;
3878 else if (VT == MVT::f64)
3879 LC = RTLIB::LLRINT_F64;
3880 else if (VT == MVT::f80)
3881 LC = RTLIB::LLRINT_F80;
3882 else if (VT == MVT::f128)
3883 LC = RTLIB::LLRINT_F128;
3884 else if (VT == MVT::ppcf128)
3885 LC = RTLIB::LLRINT_PPCF128;
3886 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
3890 EVT RetVT =
N->getValueType(0);
3894 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
3895 Op, CallOptions, dl,
3897 SplitInteger(Tmp.first,
Lo,
Hi);
3899 if (
N->isStrictFPOpcode())
3900 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3903void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
3905 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
3908 ExpandRes_NormalLoad(
N,
Lo,
Hi);
3914 EVT VT =
N->getValueType(0);
3925 if (
N->getMemoryVT().bitsLE(NVT)) {
3926 EVT MemVT =
N->getMemoryVT();
3929 N->getOriginalAlign(), MMOFlags, AAInfo);
3932 Ch =
Lo.getValue(1);
3937 unsigned LoSize =
Lo.getValueSizeInBits();
3952 N->getOriginalAlign(), MMOFlags, AAInfo);
3954 unsigned ExcessBits =
3962 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3963 N->getOriginalAlign(), MMOFlags, AAInfo);
3972 EVT MemVT =
N->getMemoryVT();
3975 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3981 N->getOriginalAlign(), MMOFlags, AAInfo);
3987 N->getPointerInfo().getWithOffset(IncrementSize),
3989 N->getOriginalAlign(), MMOFlags, AAInfo);
4013 ReplaceValueWith(
SDValue(
N, 1), Ch);
4016void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4020 GetExpandedInteger(
N->getOperand(0), LL, LH);
4021 GetExpandedInteger(
N->getOperand(1), RL, RH);
4026void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4028 EVT VT =
N->getValueType(0);
4033 GetExpandedInteger(
N->getOperand(0), LL, LH);
4034 GetExpandedInteger(
N->getOperand(1), RL, RH);
4044 LC = RTLIB::MUL_I16;
4045 else if (VT == MVT::i32)
4046 LC = RTLIB::MUL_I32;
4047 else if (VT == MVT::i64)
4048 LC = RTLIB::MUL_I64;
4049 else if (VT == MVT::i128)
4050 LC = RTLIB::MUL_I128;
4062 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4065 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4077 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4083 SplitInteger(Result,
Lo,
Hi);
4089 SplitInteger(Result,
Lo,
Hi);
4100 EVT VT =
N->getValueType(0);
4104 uint64_t Scale =
N->getConstantOperandVal(2);
4116 EVT BoolVT = getSetCCResultType(VT);
4141 SplitInteger(Result,
Lo,
Hi);
4147 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4151 GetExpandedInteger(LHS, LL, LH);
4152 GetExpandedInteger(RHS, RL, RH);
4156 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4164 SplitInteger(LoTmp, Result[0], Result[1]);
4165 SplitInteger(HiTmp, Result[2], Result[3]);
4167 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4170 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4171 "the size of the current value type");
4193 if (Scale % NVTSize) {
4209 if (Scale == VTSize)
4230 EVT BoolNVT = getSetCCResultType(NVT);
4233 if (Scale < NVTSize) {
4240 }
else if (Scale == NVTSize) {
4243 }
else if (Scale < VTSize) {
4251 "(and saturation can't happen with Scale==VTSize).");
4258 if (Scale < NVTSize) {
4263 unsigned OverflowBits = VTSize - Scale + 1;
4264 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4265 "Extent of overflow bits must start within HL");
4282 }
else if (Scale == NVTSize) {
4295 }
else if (Scale < VTSize) {
4298 unsigned OverflowBits = VTSize - Scale + 1;
4325 N->getConstantOperandVal(2), DAG);
4329 N->getConstantOperandVal(2), TLI, DAG);
4330 SplitInteger(Res,
Lo,
Hi);
4333void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4336 "Node has unexpected Opcode");
4351 SDValue LHSL, LHSH, RHSL, RHSH;
4352 GetExpandedInteger(LHS, LHSL, LHSH);
4353 GetExpandedInteger(RHS, RHSL, RHSH);
4357 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4359 Ovf =
Hi.getValue(1);
4366 SplitInteger(Sum,
Lo,
Hi);
4391 EVT VT =
LHS.getValueType();
4394 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4398 EVT OType =
Node->getValueType(1);
4406void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4408 EVT VT =
N->getValueType(0);
4410 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4420 LC = RTLIB::SDIV_I16;
4421 else if (VT == MVT::i32)
4422 LC = RTLIB::SDIV_I32;
4423 else if (VT == MVT::i64)
4424 LC = RTLIB::SDIV_I64;
4425 else if (VT == MVT::i128)
4426 LC = RTLIB::SDIV_I128;
4427 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4431 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4434void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4437 SDValue Shiftee =
N->getOperand(0);
4444 bool ShiftByByteMultiple =
4449 if (!ShiftByByteMultiple)
4453 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4454 unsigned VTByteWidth = VTBitWidth / 8;
4456 "Shiftee type size is not a power of two!");
4457 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4458 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4464 Align StackSlotAlignment(1);
4472 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4477 unsigned WideningOpc =
4479 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4486 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4492 if (ShiftByByteMultiple)
4493 Flags.setExact(
true);
4505 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4507 WillIndexUpwards = !WillIndexUpwards;
4510 if (WillIndexUpwards) {
4514 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4515 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4524 VT, dl, Ch, AdjStackPtr,
4529 if (!ShiftByByteMultiple) {
4532 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4536 SplitInteger(Res,
Lo,
Hi);
4539void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4541 EVT VT =
N->getValueType(0);
4542 unsigned Opc =
N->getOpcode();
4548 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4552 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4571 const bool LegalOrCustom =
4575 unsigned ExpansionFactor = 1;
4577 for (
EVT TmpVT = NVT;;) {
4579 if (NewTMPVT == TmpVT)
4589 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
4591 if (LegalOrCustom &&
4595 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4601 SDValue ShiftOp =
N->getOperand(1);
4606 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4608 Hi =
Lo.getValue(1);
4618 LC = RTLIB::SHL_I16;
4619 else if (VT == MVT::i32)
4620 LC = RTLIB::SHL_I32;
4621 else if (VT == MVT::i64)
4622 LC = RTLIB::SHL_I64;
4623 else if (VT == MVT::i128)
4624 LC = RTLIB::SHL_I128;
4628 LC = RTLIB::SRL_I16;
4629 else if (VT == MVT::i32)
4630 LC = RTLIB::SRL_I32;
4631 else if (VT == MVT::i64)
4632 LC = RTLIB::SRL_I64;
4633 else if (VT == MVT::i128)
4634 LC = RTLIB::SRL_I128;
4639 LC = RTLIB::SRA_I16;
4640 else if (VT == MVT::i32)
4641 LC = RTLIB::SRA_I32;
4642 else if (VT == MVT::i64)
4643 LC = RTLIB::SRA_I64;
4644 else if (VT == MVT::i128)
4645 LC = RTLIB::SRA_I128;
4652 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4655 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4659 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
4663void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4668 if (
Op.getValueType().bitsLE(NVT)) {
4679 assert(getTypeAction(
Op.getValueType()) ==
4681 "Only know how to promote this result!");
4684 "Operand over promoted?");
4686 SplitInteger(Res,
Lo,
Hi);
4694void DAGTypeLegalizer::
4697 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4698 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4720void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4722 EVT VT =
N->getValueType(0);
4724 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4734 LC = RTLIB::SREM_I16;
4735 else if (VT == MVT::i32)
4736 LC = RTLIB::SREM_I32;
4737 else if (VT == MVT::i64)
4738 LC = RTLIB::SREM_I64;
4739 else if (VT == MVT::i128)
4740 LC = RTLIB::SREM_I128;
4741 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4745 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4748void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4760void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
4762 EVT VT =
N->getValueType(0);
4781 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4782 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4783 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4785 EVT BitVT =
N->getValueType(1);
4810 SplitInteger(Three,
Lo,
Hi);
4814 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4825 LC = RTLIB::MULO_I32;
4826 else if (VT == MVT::i64)
4827 LC = RTLIB::MULO_I64;
4828 else if (VT == MVT::i128)
4829 LC = RTLIB::MULO_I128;
4842 SplitInteger(
Mul, MulLo, MulHi);
4848 SplitInteger(MulLo,
Lo,
Hi);
4849 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4862 EVT ArgVT =
Op.getValueType();
4866 Entry.IsSExt =
true;
4867 Entry.IsZExt =
false;
4868 Args.push_back(Entry);
4874 Entry.IsSExt =
true;
4875 Entry.IsZExt =
false;
4876 Args.push_back(Entry);
4895 ReplaceValueWith(
SDValue(
N, 1), Ofl);
4898void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
4900 EVT VT =
N->getValueType(0);
4902 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4911 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4914 if (isTypeLegal(NVT)) {
4916 GetExpandedInteger(
N->getOperand(0), InL, InH);
4928 LC = RTLIB::UDIV_I16;
4929 else if (VT == MVT::i32)
4930 LC = RTLIB::UDIV_I32;
4931 else if (VT == MVT::i64)
4932 LC = RTLIB::UDIV_I64;
4933 else if (VT == MVT::i128)
4934 LC = RTLIB::UDIV_I128;
4935 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
4938 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4941void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
4943 EVT VT =
N->getValueType(0);
4945 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4954 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4957 if (isTypeLegal(NVT)) {
4959 GetExpandedInteger(
N->getOperand(0), InL, InH);
4971 LC = RTLIB::UREM_I16;
4972 else if (VT == MVT::i32)
4973 LC = RTLIB::UREM_I32;
4974 else if (VT == MVT::i64)
4975 LC = RTLIB::UREM_I64;
4976 else if (VT == MVT::i128)
4977 LC = RTLIB::UREM_I128;
4978 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
4981 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4984void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
4989 if (
Op.getValueType().bitsLE(NVT)) {
4996 assert(getTypeAction(
Op.getValueType()) ==
4998 "Only know how to promote this result!");
5001 "Operand over promoted?");
5003 SplitInteger(Res,
Lo,
Hi);
5011void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5014 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5019 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5020 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5026void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5031 SplitInteger(Res,
Lo,
Hi);
5034void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5040 N->getOperand(0),
N->getOperand(1));
5041 SplitInteger(Res,
Lo,
Hi);
5048 GetExpandedInteger(
N->getOperand(0), In3, In4);
5049 GetExpandedInteger(
N->getOperand(1), In1, In2);
5053 unsigned Opc =
N->getOpcode();
5056 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5073 Lo = DAG.
getNode(Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5074 Hi = DAG.
getNode(Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5079 EVT VT =
N->getValueType(0);
5089 SplitInteger(Res,
Lo,
Hi);
5100bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5104 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5107 switch (
N->getOpcode()) {
5110 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5111 N->dump(&DAG);
dbgs() <<
"\n";
5116 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5123 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5129 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5136 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5142 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5145 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5147 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5148 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5149 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5154 if (!Res.
getNode())
return false;
5162 "Invalid operand expansion");
5164 ReplaceValueWith(
SDValue(
N, 0), Res);
5170void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5174 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5175 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5176 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5196 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5197 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5229 RHSLo, LowCC,
false, DagCombineInfo, dl);
5236 RHSHi, CCCode,
false, DagCombineInfo, dl);
5249 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5251 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5260 if (LHSHi == RHSHi) {
5273 if (HasSETCCCARRY) {
5276 bool FlipOperands =
false;
5304 false, DagCombineInfo, dl);
5313 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5314 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5315 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5327 N->getOperand(4)), 0);
5331 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5332 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5333 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5344 N->getOperand(2),
N->getOperand(3),
5349 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5350 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5351 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5356 "Unexpected setcc expansion!");
5372 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5373 GetExpandedInteger(LHS, LHSLo, LHSHi);
5374 GetExpandedInteger(RHS, RHSLo, RHSHi);
5387 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5397 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5406 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5411 bool IsStrict =
N->isStrictFPOpcode();
5415 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5416 EVT DstVT =
N->getValueType(0);
5419 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5420 "Don't know how to expand this XINT_TO_FP!");
5423 std::pair<SDValue, SDValue> Tmp =
5429 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5430 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5435 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5438 return ExpandOp_NormalStore(
N, OpNo);
5441 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5443 EVT VT =
N->getOperand(1).getValueType();
5454 if (
N->getMemoryVT().bitsLE(NVT)) {
5455 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5457 N->getMemoryVT(),
N->getOriginalAlign(), MMOFlags,
5463 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5466 N->getOriginalAlign(), MMOFlags, AAInfo);
5468 unsigned ExcessBits =
5476 N->getPointerInfo().getWithOffset(IncrementSize),
5477 NEVT,
N->getOriginalAlign(), MMOFlags, AAInfo);
5483 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5485 EVT ExtVT =
N->getMemoryVT();
5488 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5506 N->getOriginalAlign(), MMOFlags, AAInfo);
5512 N->getPointerInfo().getWithOffset(IncrementSize),
5514 N->getOriginalAlign(), MMOFlags, AAInfo);
5520 GetExpandedInteger(
N->getOperand(0), InL, InH);
5529 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5530 cast<AtomicSDNode>(
N)->getMemOperand());
5534SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5535 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5536 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5540 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5545SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5548 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5549 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5555SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
5558 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5559 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5568SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
5570 EVT OutVT =
N->getValueType(0);
5572 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5576 SDValue BaseIdx =
N->getOperand(1);
5603 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5612 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5616 "Promoted operand has an element type greater than result");
5629 InOp0 = GetPromotedInteger(
N->getOperand(0));
5636 for (
unsigned i = 0; i != OutNumElems; ++i) {
5652SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
5653 EVT OutVT =
N->getValueType(0);
5655 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5667 Vec = GetPromotedInteger(Vec);
5673SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
5676 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5682SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
5684 EVT VT =
N->getValueType(0);
5689 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5690 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5697 EVT OutVT =
N->getValueType(0);
5699 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5700 unsigned NumElems =
N->getNumOperands();
5708 for (
unsigned i = 0; i != NumElems; ++i) {
5710 EVT OpVT =
Op.getValueType();
5715 if (OpVT.
bitsLT(NOutVTElem)) {
5721 ExtOpc = NOutExtOpc;
5734 assert(!
N->getOperand(0).getValueType().isVector() &&
5735 "Input must be a scalar");
5737 EVT OutVT =
N->getValueType(0);
5739 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5744 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
5749 EVT OutVT =
N->getValueType(0);
5752 "Type must be promoted to a scalable vector type");
5753 const APInt &StepVal =
N->getConstantOperandAPInt(0);
5758SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
5761 EVT OutVT =
N->getValueType(0);
5763 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5765 unsigned NumOperands =
N->getNumOperands();
5770 SDUse *MaxSizedValue = std::max_element(
5772 EVT AVT = A.getValueType().getVectorElementType();
5773 EVT BVT = B.getValueType().getVectorElementType();
5774 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5780 for (
unsigned I = 0;
I < NumOperands; ++
I) {
5782 EVT OpVT =
Op.getValueType();
5784 Op = GetPromotedInteger(
Op);
5787 "Unhandled legalization type");
5804 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
5805 assert(NumElem * NumOperands == NumOutElem &&
5806 "Unexpected number of elements");
5810 for (
unsigned i = 0; i < NumOperands; ++i) {
5813 Op = GetPromotedInteger(
Op);
5814 EVT SclrTy =
Op.getValueType().getVectorElementType();
5815 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
5816 "Unexpected number of elements");
5818 for (
unsigned j = 0;
j < NumElem; ++
j) {
5828SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
5829 EVT VT =
N->getValueType(0);
5831 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
5839 if (getTypeAction(
N->getOperand(0).getValueType())
5843 switch(
N->getOpcode()) {
5845 Promoted = SExtPromotedInteger(
N->getOperand(0));
5848 Promoted = ZExtPromotedInteger(
N->getOperand(0));
5851 Promoted = GetPromotedInteger(
N->getOperand(0));
5856 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
5860 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
5863SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
5864 EVT OutVT =
N->getValueType(0);
5866 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5871 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5874 NOutVTElem,
N->getOperand(1));
5876 V0, ConvElem,
N->getOperand(2));
5884 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
5892 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
5893 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
5894 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
5897SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
5899 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5911SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
5916 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5926SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
5928 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5931 N->getValueType(0).getVectorNumElements());
5936SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
5939 EVT ResVT =
N->getValueType(0);
5940 unsigned NumElems =
N->getNumOperands();
5945 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5947 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
5955 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
5961 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5963 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
5964 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
5966 for (
unsigned i=0; i<NumElem; ++i) {
5978SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
5991 for (
unsigned I = 0;
I < OpNo;
I++)
5994 EVT Ty =
Op.getValueType();
6006 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6011 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6017SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6030 for (
unsigned I = 0;
I < OpNo;
I++)
6033 EVT Ty =
Op.getValueType();
6045 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6050 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)