31#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::VP_BITREVERSE:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
80 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
81 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
82 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
84 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
90 Res = PromoteIntRes_Select(
N);
95 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
97 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
99 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
102 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
104 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
106 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(
N);
break;
108 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(
N);
break;
109 case ISD::VP_TRUNCATE:
111 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
112 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
116 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
118 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
120 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
122 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
124 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
127 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
130 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
132 Res = PromoteIntRes_BUILD_VECTOR(
N);
136 Res = PromoteIntRes_ScalarOp(
N);
140 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
145 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
148 case ISD::VP_SIGN_EXTEND:
150 case ISD::VP_ZERO_EXTEND:
153 case ISD::VP_FP_TO_SINT:
154 case ISD::VP_FP_TO_UINT:
162 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
166 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
170 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
185 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
192 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
199 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
202 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
204 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
206 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
222 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
224 case ISD::VP_SADDSAT:
225 case ISD::VP_UADDSAT:
226 case ISD::VP_SSUBSAT:
227 case ISD::VP_USUBSAT:
228 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
241 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
244 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
258 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
262 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
274 Res = PromoteIntRes_VECREDUCE(
N);
277 case ISD::VP_REDUCE_ADD:
278 case ISD::VP_REDUCE_MUL:
279 case ISD::VP_REDUCE_AND:
280 case ISD::VP_REDUCE_OR:
281 case ISD::VP_REDUCE_XOR:
282 case ISD::VP_REDUCE_SMAX:
283 case ISD::VP_REDUCE_SMIN:
284 case ISD::VP_REDUCE_UMAX:
285 case ISD::VP_REDUCE_UMIN:
286 Res = PromoteIntRes_VP_REDUCE(
N);
290 Res = PromoteIntRes_FREEZE(
N);
295 Res = PromoteIntRes_Rotate(
N);
300 Res = PromoteIntRes_FunnelShift(
N);
305 Res = PromoteIntRes_VPFunnelShift(
N);
309 Res = PromoteIntRes_IS_FPCLASS(
N);
312 Res = PromoteIntRes_FFREXP(
N);
317 Res = PromoteIntRes_XRINT(
N);
323 SetPromotedInteger(
SDValue(
N, ResNo), Res);
328 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
329 return GetPromotedInteger(
Op);
334 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
336 Op.getValueType(),
Op,
N->getOperand(1));
341 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
343 Op.getValueType(),
Op,
N->getOperand(1));
349 N->getMemoryVT(), ResVT,
350 N->getChain(),
N->getBasePtr(),
369 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
379 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
382 N->getChain(),
N->getBasePtr(),
383 Op2,
N->getMemOperand());
394 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
405 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
415 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
418 Op2 = SExtPromotedInteger(Op2);
421 Op2 = ZExtPromotedInteger(Op2);
424 Op2 = GetPromotedInteger(Op2);
433 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
434 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
436 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
445 EVT OutVT =
N->getValueType(0);
449 switch (getTypeAction(InVT)) {
476 BitConvertToInteger(GetScalarizedVector(InOp)));
485 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
486 Lo = BitConvertToInteger(
Lo);
487 Hi = BitConvertToInteger(
Hi);
495 JoinIntegers(
Lo,
Hi));
529 if (isTypeLegal(WideOutVT)) {
530 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
540 CreateStackStoreLoad(InOp, OutVT));
544 SDValue V = GetPromotedInteger(
N->getOperand(0));
546 V.getValueType(), V);
550 SDValue Op = GetPromotedInteger(
N->getOperand(0));
551 EVT OVT =
N->getValueType(0);
552 EVT NVT =
Op.getValueType();
572 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
573 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
578 SDValue Op = GetPromotedInteger(
N->getOperand(0));
579 EVT OVT =
N->getValueType(0);
580 EVT NVT =
Op.getValueType();
600 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
601 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
610 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
615 EVT VT =
N->getValueType(0);
624 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
629 EVT OVT =
N->getValueType(0);
646 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
651 if (!
N->isVPOpcode())
657 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
658 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
659 ExtractLeadingBits, Mask, EVL);
663 EVT OVT =
N->getValueType(0);
680 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
681 if (!
N->isVPOpcode())
684 N->getOperand(1),
N->getOperand(2));
688 SDValue Op = GetPromotedInteger(
N->getOperand(0));
689 EVT OVT =
N->getValueType(0);
690 EVT NVT =
Op.getValueType();
708 if (
N->getOpcode() ==
ISD::CTTZ ||
N->getOpcode() == ISD::VP_CTTZ) {
719 N->getOperand(1),
N->getOperand(2));
721 if (!
N->isVPOpcode())
722 return DAG.
getNode(
N->getOpcode(), dl, NVT,
Op);
723 return DAG.
getNode(
N->getOpcode(), dl, NVT,
Op,
N->getOperand(1),
727SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
742 EVT SVT =
In.getValueType().getScalarType();
754 unsigned NewOpc =
N->getOpcode();
771 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
774 NewOpc = ISD::VP_FP_TO_SINT;
777 if (
N->isStrictFPOpcode()) {
778 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
779 {
N->getOperand(0),
N->getOperand(1)});
783 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
784 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
787 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
799 N->getOpcode() == ISD::VP_FP_TO_UINT)
806SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
810 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
814SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
818 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
821SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
826 N->getOperand(0),
N->getOperand(1));
834 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
842 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
854 if (getTypeAction(
N->getOperand(0).getValueType())
856 SDValue Res = GetPromotedInteger(
N->getOperand(0));
874 if (
N->getNumOperands() != 1) {
875 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
876 assert(
N->isVPOpcode() &&
"Expected VP opcode");
877 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
878 N->getOperand(1),
N->getOperand(2));
880 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
890 N->getMemoryVT(),
N->getMemOperand());
900 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
908 N->getOffset(),
N->getMask(), ExtPassThru,
909 N->getMemoryVT(),
N->getMemOperand(),
910 N->getAddressingMode(), ExtType,
911 N->isExpandingLoad());
920 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
922 "Gather result type and the passThru argument type should be the same");
929 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
930 N->getIndex(),
N->getScale() };
932 N->getMemoryVT(), dl, Ops,
933 N->getMemOperand(),
N->getIndexType(),
946 EVT VT =
N->getValueType(0);
947 EVT SVT = getSetCCResultType(VT);
948 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
949 unsigned NumOps =
N->getNumOperands();
950 assert(NumOps <= 3 &&
"Too many operands");
952 Ops[2] =
N->getOperand(2);
960 ReplaceValueWith(
SDValue(
N, 0), Res);
966template <
class MatchContextClass>
978 MatchContextClass matcher(DAG, TLI,
N);
981 unsigned Opcode = matcher.getRootBaseOpcode();
985 SDValue Op1Promoted, Op2Promoted;
987 Op1Promoted = GetPromotedInteger(Op1);
988 Op2Promoted = ZExtPromotedInteger(Op2);
990 Op1Promoted = ZExtPromotedInteger(Op1);
991 Op2Promoted = ZExtPromotedInteger(Op2);
993 Op1Promoted = SExtPromotedInteger(Op1);
994 Op2Promoted = SExtPromotedInteger(Op2);
1003 matcher.getNode(
ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1004 return matcher.getNode(
ISD::UMIN, dl, PromotedType,
Add, SatMax);
1009 return matcher.getNode(
ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1014 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1027 "addition, subtraction or left shift");
1030 unsigned SHLAmount = NewBits - OldBits;
1037 matcher.
getNode(
ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1040 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1041 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1050 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1051 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1052 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1059 SDValue Op1Promoted, Op2Promoted;
1065 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1066 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1068 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1069 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1071 EVT OldType =
N->getOperand(0).getValueType();
1085 Op2Promoted,
N->getOperand(2));
1087 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1090 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1095 unsigned SatW,
bool Signed,
1098 EVT VT = V.getValueType();
1125 EVT VT =
LHS.getValueType();
1143 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1149 "Tried to saturate to more than the original type?");
1158 SDValue Op1Promoted, Op2Promoted;
1164 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1165 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1167 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1168 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1171 unsigned Scale =
N->getConstantOperandVal(2);
1180 N->getValueType(0).getScalarSizeInBits();
1185 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1186 Op2Promoted,
N->getOperand(2));
1196 Op2Promoted, Scale, DAG)) {
1199 N->getValueType(0).getScalarSizeInBits(),
1207 N->getValueType(0).getScalarSizeInBits());
1210SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1212 return PromoteIntRes_Overflow(
N);
1216 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1217 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1218 EVT OVT =
N->getOperand(0).getValueType();
1219 EVT NVT =
LHS.getValueType();
1234 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1242 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1243 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1245 unsigned Opcode =
N->getOpcode();
1246 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1253 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1254 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1256 LHS.getValueType(),
N->getOperand(0),
1257 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1261 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1262 EVT InVT =
N->getOperand(OpNo).getValueType();
1265 EVT SVT = getSetCCResultType(InVT);
1273 SVT = getSetCCResultType(InVT);
1281 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1282 "Vector compare must return a vector result!");
1286 if (
N->isStrictFPOpcode()) {
1288 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1289 N->getOperand(2),
N->getOperand(3)};
1290 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1295 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1296 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1312 EVT VT =
N->getValueType(0);
1318 ReplaceValueWith(
SDValue(
N, 0), Res);
1323 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1326 RHS = ZExtPromotedInteger(RHS);
1327 if (
N->getOpcode() != ISD::VP_SHL)
1330 N->getOperand(2),
N->getOperand(3));
1333SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1334 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1336 Op.getValueType(),
Op,
N->getOperand(1));
1339SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1343 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1344 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1345 if (
N->getNumOperands() == 2)
1347 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1348 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1350 N->getOperand(2),
N->getOperand(3));
1355 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1356 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1357 if (
N->getNumOperands() == 2)
1359 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1360 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1362 N->getOperand(2),
N->getOperand(3));
1367 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1368 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1369 if (
N->getNumOperands() == 2)
1371 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1372 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1374 N->getOperand(2),
N->getOperand(3));
1383 SExtOrZExtPromotedOperands(LHS, RHS);
1386 LHS.getValueType(), LHS, RHS);
1391 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1394 RHS = ZExtPromotedInteger(RHS);
1395 if (
N->getOpcode() != ISD::VP_ASHR)
1398 N->getOperand(2),
N->getOperand(3));
1403 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1406 RHS = ZExtPromotedInteger(RHS);
1407 if (
N->getOpcode() != ISD::VP_LSHR)
1410 N->getOperand(2),
N->getOperand(3));
1416 ReplaceValueWith(
SDValue(
N, 0), Res);
1421 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1422 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1425 Amt = ZExtPromotedInteger(Amt);
1429 EVT OldVT =
N->getOperand(0).getValueType();
1430 EVT VT =
Lo.getValueType();
1431 unsigned Opcode =
N->getOpcode();
1445 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1470SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1471 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1472 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1477 Amt = ZExtPromotedInteger(Amt);
1481 EVT OldVT =
N->getOperand(0).getValueType();
1482 EVT VT =
Lo.getValueType();
1483 unsigned Opcode =
N->getOpcode();
1484 bool IsFSHR = Opcode == ISD::VP_FSHR;
1489 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1497 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1500 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1504 Res = DAG.
getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL,
DL, VT, Res, Amt,
1507 Res = DAG.
getNode(ISD::VP_LSHR,
DL, VT, Res, HiShift, Mask, EVL);
1513 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1518 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1536 Res = GetPromotedInteger(InOp);
1543 "Dst and Src must have the same number of elements");
1545 "Promoted vector type must be a power of two");
1548 GetSplitVector(InOp, EOp1, EOp2);
1556 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1557 "Expected VP_TRUNCATE opcode");
1558 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1559 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1560 std::tie(EVLLo, EVLHi) =
1561 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1562 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1563 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1570 SDValue WideInOp = GetWidenedVector(InOp);
1575 N->getValueType(0).getScalarType(), NumElem);
1590 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1591 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1596SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1598 return PromoteIntRes_Overflow(
N);
1602 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1603 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1604 EVT OVT =
N->getOperand(0).getValueType();
1605 EVT NVT =
LHS.getValueType();
1619 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1630 return PromoteIntRes_Overflow(
N);
1642 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1643 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1645 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1649 LHS, RHS,
N->getOperand(2));
1659 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1660 return PromoteIntRes_Overflow(
N);
1664 EVT OVT =
N->getValueType(0);
1677 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1681SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1684 return PromoteIntRes_Overflow(
N);
1688 EVT SmallVT =
LHS.getValueType();
1695 LHS = SExtPromotedInteger(LHS);
1696 RHS = SExtPromotedInteger(RHS);
1698 LHS = ZExtPromotedInteger(LHS);
1699 RHS = ZExtPromotedInteger(RHS);
1730 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1736 N->getValueType(0)));
1742 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1749 EVT VT =
N->getValueType(0);
1757 for (
unsigned i = 0; i < NumRegs; ++i) {
1758 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1759 N->getConstantOperandVal(3));
1765 std::reverse(Parts.begin(), Parts.end());
1770 for (
unsigned i = 1; i < NumRegs; ++i) {
1781 ReplaceValueWith(
SDValue(
N, 1), Chain);
1794bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1797 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1802 switch (
N->getOpcode()) {
1805 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1806 N->dump(&DAG);
dbgs() <<
"\n";
1812 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1815 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1816 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1822 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1826 Res = PromoteIntOp_ScalarOp(
N);
1829 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1832 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1834 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
1835 case ISD::VP_SINT_TO_FP:
1838 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1840 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1842 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1844 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1846 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1848 case ISD::VP_TRUNCATE:
1852 case ISD::VP_UINT_TO_FP:
1857 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
1865 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1868 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1899 case ISD::VP_REDUCE_ADD:
1900 case ISD::VP_REDUCE_MUL:
1901 case ISD::VP_REDUCE_AND:
1902 case ISD::VP_REDUCE_OR:
1903 case ISD::VP_REDUCE_XOR:
1904 case ISD::VP_REDUCE_SMAX:
1905 case ISD::VP_REDUCE_SMIN:
1906 case ISD::VP_REDUCE_UMAX:
1907 case ISD::VP_REDUCE_UMIN:
1908 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
1913 Res = PromoteIntOp_STACKMAP(
N, OpNo);
1916 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
1918 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1919 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1920 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
1922 case ISD::EXPERIMENTAL_VP_SPLICE:
1923 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
1928 if (!Res.
getNode())
return false;
1935 const bool IsStrictFp =
N->isStrictFPOpcode();
1937 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1938 "Invalid operand expansion");
1942 ReplaceValueWith(
SDValue(
N, 0), Res);
1952void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
1953 SDValue OpL = GetPromotedInteger(LHS);
1954 SDValue OpR = GetPromotedInteger(RHS);
1960 unsigned OpLEffectiveBits =
1962 unsigned OpREffectiveBits =
1964 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1965 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1972 LHS = SExtPromotedInteger(LHS);
1973 RHS = SExtPromotedInteger(RHS);
1984 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1985 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1992 LHS = ZExtPromotedInteger(LHS);
1993 RHS = ZExtPromotedInteger(RHS);
1998void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2007 LHS = SExtPromotedInteger(LHS);
2008 RHS = SExtPromotedInteger(RHS);
2013 "Unknown integer comparison!");
2015 SExtOrZExtPromotedOperands(LHS, RHS);
2019 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2024 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2026 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2032 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
2035SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2036 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2040 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2045 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2049SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2050 assert(OpNo == 1 &&
"only know how to promote condition");
2053 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2057 N->getOperand(2)), 0);
2062 EVT OVT =
N->getOperand(0).getValueType();
2063 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2064 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2065 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2078 EVT VecVT =
N->getValueType(0);
2081 "Legal vector of one illegal element?");
2086 assert(
N->getOperand(0).getValueSizeInBits() >=
2087 N->getValueType(0).getScalarSizeInBits() &&
2088 "Type of inserted value narrower than vector element type!");
2091 for (
unsigned i = 0; i < NumElts; ++i)
2092 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2097SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2104 assert(
N->getOperand(1).getValueSizeInBits() >=
2105 N->getValueType(0).getScalarSizeInBits() &&
2106 "Type of inserted value narrower than vector element type!");
2108 GetPromotedInteger(
N->getOperand(1)),
2113 assert(OpNo == 2 &&
"Different operand and result vector types?");
2119 N->getOperand(1),
Idx), 0);
2126 GetPromotedInteger(
N->getOperand(0))), 0);
2129SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2130 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2132 EVT OpTy =
N->getOperand(1).getValueType();
2135 if (
SDValue Res = WidenVSELECTMask(
N))
2137 Res,
N->getOperand(1),
N->getOperand(2));
2141 Cond = PromoteTargetBoolean(
Cond, OpVT);
2144 N->getOperand(2)), 0);
2147SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2148 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2152 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2156 N->getOperand(3),
N->getOperand(4)), 0);
2159SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2160 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2164 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2170 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2173 N->getOperand(3),
N->getOperand(4)),
2179 ZExtPromotedInteger(
N->getOperand(1))), 0);
2184 ZExtPromotedInteger(
N->getOperand(2))), 0);
2188 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2195SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2197 EVT VT =
N->getValueType(0);
2198 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2200 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2208 return DAG.
getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt,
N->getOperand(1),
2213 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2215 SExtPromotedInteger(
N->getOperand(0)),
2216 N->getOperand(1),
N->getOperand(2)),
2219 SExtPromotedInteger(
N->getOperand(0))), 0);
2222SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2224 SExtPromotedInteger(
N->getOperand(1))), 0);
2232 SDValue Val = GetPromotedInteger(
N->getValue());
2236 N->getMemoryVT(),
N->getMemOperand());
2247 Mask = PromoteTargetBoolean(Mask, DataVT);
2253 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2254 DataOp = GetPromotedInteger(DataOp);
2257 N->getOffset(), Mask,
N->getMemoryVT(),
2258 N->getMemOperand(),
N->getAddressingMode(),
2259 true,
N->isCompressingStore());
2264 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2265 EVT DataVT =
N->getValueType(0);
2266 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2268 NewOps[OpNo] =
Mask;
2285 EVT DataVT =
N->getValueType(0);
2286 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2287 }
else if (OpNo == 4) {
2289 if (
N->isIndexSigned())
2291 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2293 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2295 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2309 bool TruncateStore =
N->isTruncatingStore();
2314 EVT DataVT =
N->getValue().getValueType();
2315 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2316 }
else if (OpNo == 4) {
2318 if (
N->isIndexSigned())
2320 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2322 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2324 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2325 TruncateStore =
true;
2329 SDLoc(
N), NewOps,
N->getMemOperand(),
2330 N->getIndexType(), TruncateStore);
2334 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2335 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2337 N->getOperand(1),
N->getOperand(2));
2342 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2344 ZExtPromotedInteger(
N->getOperand(0)),
2345 N->getOperand(1),
N->getOperand(2)),
2348 ZExtPromotedInteger(
N->getOperand(0))), 0);
2351SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2353 ZExtPromotedInteger(
N->getOperand(1))), 0);
2358 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2363SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2365 EVT VT =
N->getValueType(0);
2366 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2368 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2371 N->getOperand(0).getScalarValueSizeInBits());
2373 N->getOperand(1),
N->getOperand(2));
2376SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(
SDNode *
N,
unsigned OpNo) {
2377 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2384 Carry = PromoteTargetBoolean(Carry,
LHS.getValueType());
2390 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2395SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2397 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2402 bool IsStrict =
N->isStrictFPOpcode();
2414 SDValue Op = SExtPromotedInteger(
N->getOperand(1));
2424 unsigned OpOffset = IsStrict ? 1 : 0;
2427 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2428 "POWI exponent should match with sizeof(int) when doing the libcall.");
2431 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2432 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2433 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2434 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2436 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2441 switch (
N->getOpcode()) {
2449 case ISD::VP_REDUCE_ADD:
2450 case ISD::VP_REDUCE_MUL:
2451 case ISD::VP_REDUCE_AND:
2452 case ISD::VP_REDUCE_OR:
2453 case ISD::VP_REDUCE_XOR:
2457 case ISD::VP_REDUCE_SMAX:
2458 case ISD::VP_REDUCE_SMIN:
2462 case ISD::VP_REDUCE_UMAX:
2463 case ISD::VP_REDUCE_UMIN:
2473 return GetPromotedInteger(V);
2475 return SExtPromotedInteger(V);
2477 return ZExtPromotedInteger(V);
2483 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2485 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2486 EVT InVT =
Op.getValueType();
2488 EVT ResVT =
N->getValueType(0);
2489 unsigned Opcode =
N->getOpcode();
2509 Op = ZExtPromotedInteger(
N->getOperand(0));
2512 Op = SExtPromotedInteger(
N->getOperand(0));
2528 Op = ZExtPromotedInteger(
N->getOperand(0));
2531 Op = SExtPromotedInteger(
N->getOperand(0));
2545SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2552 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2556 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2558 Op = PromoteIntOpVectorReduction(
N,
Op);
2562 EVT VT =
N->getValueType(0);
2563 EVT EltVT =
Op.getValueType().getScalarType();
2578 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2582SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2585 SDValue Operand =
N->getOperand(OpNo);
2591SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2594 SDValue Operand =
N->getOperand(OpNo);
2600SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2601 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2602 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2605 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2610SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2614 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2618 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2620 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2632void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2638 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2641 switch (
N->getOpcode()) {
2644 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2645 N->dump(&DAG);
dbgs() <<
"\n";
2691 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2718 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2719 SplitInteger(Tmp.first,
Lo,
Hi);
2720 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2728 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2737 SplitInteger(Tmp,
Lo,
Hi);
2808 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2813 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2817 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2827std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2828 unsigned Opc =
Node->getOpcode();
2834 EVT RetVT =
Node->getValueType(0);
2842 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2843 "Unexpected atomic op or value type!");
2847 Node->getOperand(0));
2852void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2857 GetExpandedInteger(
N->getOperand(0), InL, InH);
2868 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2872 if (Amt.
uge(VTBits)) {
2874 }
else if (Amt.
ugt(NVTBits)) {
2878 }
else if (Amt == NVTBits) {
2895 if (Amt.
uge(VTBits)) {
2897 }
else if (Amt.
ugt(NVTBits)) {
2901 }
else if (Amt == NVTBits) {
2918 if (Amt.
uge(VTBits)) {
2921 }
else if (Amt.
ugt(NVTBits)) {
2926 }
else if (Amt == NVTBits) {
2946bool DAGTypeLegalizer::
2948 unsigned Opc =
N->getOpcode();
2956 "Expanded integer type size not a power of two!");
2963 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
2968 GetExpandedInteger(In, InL, InH);
3022 Lo = DAG.
getNode(Opc, dl, NVT, InL, Amt);
3035bool DAGTypeLegalizer::
3042 "Expanded integer type size not a power of two!");
3047 GetExpandedInteger(
N->getOperand(0), InL, InH);
3059 switch (
N->getOpcode()) {
3074 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3090 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3106 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3127void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3136 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3137 unsigned NumHalfBits = NumBits / 2;
3140 SDValue LHSL, LHSH, RHSL, RHSH;
3141 GetExpandedInteger(LHS, LHSL, LHSH);
3142 GetExpandedInteger(RHS, RHSL, RHSH);
3155 SDValue LHSL, LHSH, RHSL, RHSH;
3156 GetExpandedInteger(LHS, LHSL, LHSH);
3157 GetExpandedInteger(RHS, RHSL, RHSH);
3159 EVT CCT = getSetCCResultType(NVT);
3168 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3172 const APInt *RHSVal =
nullptr;
3173 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3174 RHSVal = &RHSConst->getAPIntValue();
3181 SDValue LHSL, LHSH, RHSL, RHSH;
3182 GetExpandedInteger(LHS, LHSL, LHSH);
3183 GetExpandedInteger(RHS, RHSL, RHSH);
3185 EVT CCT = getSetCCResultType(NVT);
3191 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3210 switch (
N->getOpcode()) {
3237 EVT VT =
N->getValueType(0);
3238 EVT CCT = getSetCCResultType(VT);
3241 SplitInteger(Result,
Lo,
Hi);
3244void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3248 SDValue LHSL, LHSH, RHSL, RHSH;
3249 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3250 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3253 SDValue LoOps[2] = { LHSL, RHSL };
3254 SDValue HiOps[3] = { LHSH, RHSH };
3263 HiOps[2] =
Lo.getValue(1);
3269 HiOps[2] =
Lo.getValue(1);
3291 HiOps[2] =
Lo.getValue(1);
3295 HiOps[2] =
Lo.getValue(1);
3308 EVT OvfVT = getSetCCResultType(NVT);
3348 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3351 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3354 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3372 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3386void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3389 SDValue LHSL, LHSH, RHSL, RHSH;
3391 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3392 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3394 SDValue LoOps[2] = { LHSL, RHSL };
3395 SDValue HiOps[3] = { LHSH, RHSH };
3399 HiOps[2] =
Lo.getValue(1);
3403 HiOps[2] =
Lo.getValue(1);
3409 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3412void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3415 SDValue LHSL, LHSH, RHSL, RHSH;
3417 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3418 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3420 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3421 SDValue HiOps[3] = { LHSH, RHSH };
3423 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3424 HiOps[2] =
Lo.getValue(1);
3425 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3429 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3432void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3440 unsigned CarryOp, NoCarryOp;
3442 switch(
N->getOpcode()) {
3462 SDValue LHSL, LHSH, RHSL, RHSH;
3463 GetExpandedInteger(LHS, LHSL, LHSH);
3464 GetExpandedInteger(RHS, RHSL, RHSH);
3466 SDValue LoOps[2] = { LHSL, RHSL };
3467 SDValue HiOps[3] = { LHSH, RHSH };
3469 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3470 HiOps[2] =
Lo.getValue(1);
3471 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3473 Ovf =
Hi.getValue(1);
3478 SplitInteger(Sum,
Lo,
Hi);
3489 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3500 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3506 SDValue LHSL, LHSH, RHSL, RHSH;
3508 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3509 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3511 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3514 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3515 HiOps[2] =
Lo.getValue(1);
3516 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3520 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3523void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3526 SDValue LHSL, LHSH, RHSL, RHSH;
3528 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3529 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3535 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3536 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3540 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3543void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3548 if (
Op.getValueType().bitsLE(NVT)) {
3555 assert(getTypeAction(
Op.getValueType()) ==
3557 "Only know how to promote this result!");
3560 "Operand over promoted?");
3562 SplitInteger(Res,
Lo,
Hi);
3566void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3569 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3570 EVT NVT =
Lo.getValueType();
3571 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3575 if (NVTBits < EVTBits) {
3578 EVTBits - NVTBits)));
3588void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3591 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3592 EVT NVT =
Lo.getValueType();
3593 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3597 if (NVTBits < EVTBits) {
3600 EVTBits - NVTBits)));
3608void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3611 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3616void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3619 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3628 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3629 EVT NVT =
Lo.getValueType();
3635void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3639 auto Constant = cast<ConstantSDNode>(
N);
3641 bool IsTarget =
Constant->isTargetOpcode();
3642 bool IsOpaque =
Constant->isOpaque();
3653 GetExpandedInteger(N0,
Lo,
Hi);
3654 EVT NVT =
Lo.getValueType();
3685 EVT VT =
N->getValueType(0);
3689 SplitInteger(Neg, NegLo, NegHi);
3697void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3701 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3702 EVT NVT =
Lo.getValueType();
3717void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3721 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3722 EVT NVT =
Lo.getValueType();
3728void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3732 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3733 EVT NVT =
Lo.getValueType();
3762 ReplaceValueWith(
SDValue(
N, 1), Chain);
3770 Chain =
Op.getValue(1);
3779 EVT VT =
N->getValueType(0);
3783 bool IsStrict =
N->isStrictFPOpcode();
3785 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3787 Op = GetPromotedFloat(
Op);
3790 EVT OFPVT =
Op.getValueType();
3792 Op = GetSoftPromotedHalf(
Op);
3796 SplitInteger(
Op,
Lo,
Hi);
3800 if (
Op.getValueType() == MVT::bf16) {
3807 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
3810 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3811 CallOptions, dl, Chain);
3812 SplitInteger(Tmp.first,
Lo,
Hi);
3815 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3821 SplitInteger(Res,
Lo,
Hi);
3827 bool IsStrict =
N->isStrictFPOpcode();
3828 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3832 "Input type needs to be promoted!");
3834 EVT VT =
Op.getValueType();
3836 if (VT == MVT::f16) {
3846 LC = RTLIB::LROUND_F32;
3847 else if (VT == MVT::f64)
3848 LC = RTLIB::LROUND_F64;
3849 else if (VT == MVT::f80)
3850 LC = RTLIB::LROUND_F80;
3851 else if (VT == MVT::f128)
3852 LC = RTLIB::LROUND_F128;
3853 else if (VT == MVT::ppcf128)
3854 LC = RTLIB::LROUND_PPCF128;
3855 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3859 LC = RTLIB::LRINT_F32;
3860 else if (VT == MVT::f64)
3861 LC = RTLIB::LRINT_F64;
3862 else if (VT == MVT::f80)
3863 LC = RTLIB::LRINT_F80;
3864 else if (VT == MVT::f128)
3865 LC = RTLIB::LRINT_F128;
3866 else if (VT == MVT::ppcf128)
3867 LC = RTLIB::LRINT_PPCF128;
3868 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
3872 LC = RTLIB::LLROUND_F32;
3873 else if (VT == MVT::f64)
3874 LC = RTLIB::LLROUND_F64;
3875 else if (VT == MVT::f80)
3876 LC = RTLIB::LLROUND_F80;
3877 else if (VT == MVT::f128)
3878 LC = RTLIB::LLROUND_F128;
3879 else if (VT == MVT::ppcf128)
3880 LC = RTLIB::LLROUND_PPCF128;
3881 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3885 LC = RTLIB::LLRINT_F32;
3886 else if (VT == MVT::f64)
3887 LC = RTLIB::LLRINT_F64;
3888 else if (VT == MVT::f80)
3889 LC = RTLIB::LLRINT_F80;
3890 else if (VT == MVT::f128)
3891 LC = RTLIB::LLRINT_F128;
3892 else if (VT == MVT::ppcf128)
3893 LC = RTLIB::LLRINT_PPCF128;
3894 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
3898 EVT RetVT =
N->getValueType(0);
3902 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
3903 Op, CallOptions, dl,
3905 SplitInteger(Tmp.first,
Lo,
Hi);
3907 if (
N->isStrictFPOpcode())
3908 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3911void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
3913 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
3916 ExpandRes_NormalLoad(
N,
Lo,
Hi);
3922 EVT VT =
N->getValueType(0);
3933 if (
N->getMemoryVT().bitsLE(NVT)) {
3934 EVT MemVT =
N->getMemoryVT();
3937 N->getOriginalAlign(), MMOFlags, AAInfo);
3940 Ch =
Lo.getValue(1);
3945 unsigned LoSize =
Lo.getValueSizeInBits();
3960 N->getOriginalAlign(), MMOFlags, AAInfo);
3962 unsigned ExcessBits =
3970 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3971 N->getOriginalAlign(), MMOFlags, AAInfo);
3980 EVT MemVT =
N->getMemoryVT();
3983 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3989 N->getOriginalAlign(), MMOFlags, AAInfo);
3995 N->getPointerInfo().getWithOffset(IncrementSize),
3997 N->getOriginalAlign(), MMOFlags, AAInfo);
4021 ReplaceValueWith(
SDValue(
N, 1), Ch);
4024void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4028 GetExpandedInteger(
N->getOperand(0), LL, LH);
4029 GetExpandedInteger(
N->getOperand(1), RL, RH);
4034void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4036 EVT VT =
N->getValueType(0);
4041 GetExpandedInteger(
N->getOperand(0), LL, LH);
4042 GetExpandedInteger(
N->getOperand(1), RL, RH);
4052 LC = RTLIB::MUL_I16;
4053 else if (VT == MVT::i32)
4054 LC = RTLIB::MUL_I32;
4055 else if (VT == MVT::i64)
4056 LC = RTLIB::MUL_I64;
4057 else if (VT == MVT::i128)
4058 LC = RTLIB::MUL_I128;
4070 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4073 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4085 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4091 SplitInteger(Result,
Lo,
Hi);
4097 SplitInteger(Result,
Lo,
Hi);
4108 EVT VT =
N->getValueType(0);
4112 uint64_t Scale =
N->getConstantOperandVal(2);
4124 EVT BoolVT = getSetCCResultType(VT);
4149 SplitInteger(Result,
Lo,
Hi);
4155 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4159 GetExpandedInteger(LHS, LL, LH);
4160 GetExpandedInteger(RHS, RL, RH);
4164 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4172 SplitInteger(LoTmp, Result[0], Result[1]);
4173 SplitInteger(HiTmp, Result[2], Result[3]);
4175 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4178 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4179 "the size of the current value type");
4201 if (Scale % NVTSize) {
4217 if (Scale == VTSize)
4238 EVT BoolNVT = getSetCCResultType(NVT);
4241 if (Scale < NVTSize) {
4248 }
else if (Scale == NVTSize) {
4251 }
else if (Scale < VTSize) {
4259 "(and saturation can't happen with Scale==VTSize).");
4266 if (Scale < NVTSize) {
4271 unsigned OverflowBits = VTSize - Scale + 1;
4272 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4273 "Extent of overflow bits must start within HL");
4290 }
else if (Scale == NVTSize) {
4303 }
else if (Scale < VTSize) {
4306 unsigned OverflowBits = VTSize - Scale + 1;
4333 N->getConstantOperandVal(2), DAG);
4337 N->getConstantOperandVal(2), TLI, DAG);
4338 SplitInteger(Res,
Lo,
Hi);
4341void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4344 "Node has unexpected Opcode");
4359 SDValue LHSL, LHSH, RHSL, RHSH;
4360 GetExpandedInteger(LHS, LHSL, LHSH);
4361 GetExpandedInteger(RHS, RHSL, RHSH);
4365 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4367 Ovf =
Hi.getValue(1);
4374 SplitInteger(Sum,
Lo,
Hi);
4399 EVT VT =
LHS.getValueType();
4402 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4406 EVT OType =
Node->getValueType(1);
4414void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4416 EVT VT =
N->getValueType(0);
4418 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4428 LC = RTLIB::SDIV_I16;
4429 else if (VT == MVT::i32)
4430 LC = RTLIB::SDIV_I32;
4431 else if (VT == MVT::i64)
4432 LC = RTLIB::SDIV_I64;
4433 else if (VT == MVT::i128)
4434 LC = RTLIB::SDIV_I128;
4435 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4439 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4442void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4445 SDValue Shiftee =
N->getOperand(0);
4452 bool ShiftByByteMultiple =
4457 if (!ShiftByByteMultiple)
4461 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4462 unsigned VTByteWidth = VTBitWidth / 8;
4464 "Shiftee type size is not a power of two!");
4465 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4466 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4472 Align StackSlotAlignment(1);
4480 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4485 unsigned WideningOpc =
4487 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4494 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4500 if (ShiftByByteMultiple)
4501 Flags.setExact(
true);
4513 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4515 WillIndexUpwards = !WillIndexUpwards;
4518 if (WillIndexUpwards) {
4522 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4523 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4532 VT, dl, Ch, AdjStackPtr,
4537 if (!ShiftByByteMultiple) {
4540 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4544 SplitInteger(Res,
Lo,
Hi);
4547void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4549 EVT VT =
N->getValueType(0);
4550 unsigned Opc =
N->getOpcode();
4556 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4560 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4579 const bool LegalOrCustom =
4583 unsigned ExpansionFactor = 1;
4585 for (
EVT TmpVT = NVT;;) {
4587 if (NewTMPVT == TmpVT)
4597 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
4599 if (LegalOrCustom &&
4603 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4609 SDValue ShiftOp =
N->getOperand(1);
4614 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4616 Hi =
Lo.getValue(1);
4626 LC = RTLIB::SHL_I16;
4627 else if (VT == MVT::i32)
4628 LC = RTLIB::SHL_I32;
4629 else if (VT == MVT::i64)
4630 LC = RTLIB::SHL_I64;
4631 else if (VT == MVT::i128)
4632 LC = RTLIB::SHL_I128;
4636 LC = RTLIB::SRL_I16;
4637 else if (VT == MVT::i32)
4638 LC = RTLIB::SRL_I32;
4639 else if (VT == MVT::i64)
4640 LC = RTLIB::SRL_I64;
4641 else if (VT == MVT::i128)
4642 LC = RTLIB::SRL_I128;
4647 LC = RTLIB::SRA_I16;
4648 else if (VT == MVT::i32)
4649 LC = RTLIB::SRA_I32;
4650 else if (VT == MVT::i64)
4651 LC = RTLIB::SRA_I64;
4652 else if (VT == MVT::i128)
4653 LC = RTLIB::SRA_I128;
4660 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4663 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4667 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
4671void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4676 if (
Op.getValueType().bitsLE(NVT)) {
4687 assert(getTypeAction(
Op.getValueType()) ==
4689 "Only know how to promote this result!");
4692 "Operand over promoted?");
4694 SplitInteger(Res,
Lo,
Hi);
4702void DAGTypeLegalizer::
4705 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4706 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4728void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4730 EVT VT =
N->getValueType(0);
4732 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4742 LC = RTLIB::SREM_I16;
4743 else if (VT == MVT::i32)
4744 LC = RTLIB::SREM_I32;
4745 else if (VT == MVT::i64)
4746 LC = RTLIB::SREM_I64;
4747 else if (VT == MVT::i128)
4748 LC = RTLIB::SREM_I128;
4749 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4753 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4756void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4768void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
4770 EVT VT =
N->getValueType(0);
4789 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4790 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4791 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4793 EVT BitVT =
N->getValueType(1);
4818 SplitInteger(Three,
Lo,
Hi);
4822 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4833 LC = RTLIB::MULO_I32;
4834 else if (VT == MVT::i64)
4835 LC = RTLIB::MULO_I64;
4836 else if (VT == MVT::i128)
4837 LC = RTLIB::MULO_I128;
4850 SplitInteger(
Mul, MulLo, MulHi);
4856 SplitInteger(MulLo,
Lo,
Hi);
4857 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4870 EVT ArgVT =
Op.getValueType();
4874 Entry.IsSExt =
true;
4875 Entry.IsZExt =
false;
4876 Args.push_back(Entry);
4882 Entry.IsSExt =
true;
4883 Entry.IsZExt =
false;
4884 Args.push_back(Entry);
4903 ReplaceValueWith(
SDValue(
N, 1), Ofl);
4906void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
4908 EVT VT =
N->getValueType(0);
4910 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4919 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4922 if (isTypeLegal(NVT)) {
4924 GetExpandedInteger(
N->getOperand(0), InL, InH);
4936 LC = RTLIB::UDIV_I16;
4937 else if (VT == MVT::i32)
4938 LC = RTLIB::UDIV_I32;
4939 else if (VT == MVT::i64)
4940 LC = RTLIB::UDIV_I64;
4941 else if (VT == MVT::i128)
4942 LC = RTLIB::UDIV_I128;
4943 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
4946 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4949void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
4951 EVT VT =
N->getValueType(0);
4953 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4962 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4965 if (isTypeLegal(NVT)) {
4967 GetExpandedInteger(
N->getOperand(0), InL, InH);
4979 LC = RTLIB::UREM_I16;
4980 else if (VT == MVT::i32)
4981 LC = RTLIB::UREM_I32;
4982 else if (VT == MVT::i64)
4983 LC = RTLIB::UREM_I64;
4984 else if (VT == MVT::i128)
4985 LC = RTLIB::UREM_I128;
4986 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
4989 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4992void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
4997 if (
Op.getValueType().bitsLE(NVT)) {
5004 assert(getTypeAction(
Op.getValueType()) ==
5006 "Only know how to promote this result!");
5009 "Operand over promoted?");
5011 SplitInteger(Res,
Lo,
Hi);
5019void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5022 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5027 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5028 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5034void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5039 SplitInteger(Res,
Lo,
Hi);
5042void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5048 N->getOperand(0),
N->getOperand(1));
5049 SplitInteger(Res,
Lo,
Hi);
5056 GetExpandedInteger(
N->getOperand(0), In3, In4);
5057 GetExpandedInteger(
N->getOperand(1), In1, In2);
5061 unsigned Opc =
N->getOpcode();
5064 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5081 Lo = DAG.
getNode(Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5082 Hi = DAG.
getNode(Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5087 EVT VT =
N->getValueType(0);
5097 SplitInteger(Res,
Lo,
Hi);
5108bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5112 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5115 switch (
N->getOpcode()) {
5118 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5119 N->dump(&DAG);
dbgs() <<
"\n";
5124 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5131 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5137 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5144 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5150 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5153 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5155 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5156 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5157 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5162 if (!Res.
getNode())
return false;
5170 "Invalid operand expansion");
5172 ReplaceValueWith(
SDValue(
N, 0), Res);
5178void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5182 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5183 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5184 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5204 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5205 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5237 RHSLo, LowCC,
false, DagCombineInfo, dl);
5244 RHSHi, CCCode,
false, DagCombineInfo, dl);
5257 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5259 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5268 if (LHSHi == RHSHi) {
5281 if (HasSETCCCARRY) {
5284 bool FlipOperands =
false;
5312 false, DagCombineInfo, dl);
5321 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5322 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5323 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5335 N->getOperand(4)), 0);
5339 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5340 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5341 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5352 N->getOperand(2),
N->getOperand(3),
5357 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5358 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5359 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5364 "Unexpected setcc expansion!");
5380 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5381 GetExpandedInteger(LHS, LHSLo, LHSHi);
5382 GetExpandedInteger(RHS, RHSLo, RHSHi);
5395 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5405 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5414 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5419 bool IsStrict =
N->isStrictFPOpcode();
5423 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5424 EVT DstVT =
N->getValueType(0);
5427 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5428 "Don't know how to expand this XINT_TO_FP!");
5431 std::pair<SDValue, SDValue> Tmp =
5437 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5438 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5443 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5446 return ExpandOp_NormalStore(
N, OpNo);
5449 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5451 EVT VT =
N->getOperand(1).getValueType();
5462 if (
N->getMemoryVT().bitsLE(NVT)) {
5463 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5465 N->getMemoryVT(),
N->getOriginalAlign(), MMOFlags,
5471 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5474 N->getOriginalAlign(), MMOFlags, AAInfo);
5476 unsigned ExcessBits =
5484 N->getPointerInfo().getWithOffset(IncrementSize),
5485 NEVT,
N->getOriginalAlign(), MMOFlags, AAInfo);
5491 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5493 EVT ExtVT =
N->getMemoryVT();
5496 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5514 N->getOriginalAlign(), MMOFlags, AAInfo);
5520 N->getPointerInfo().getWithOffset(IncrementSize),
5522 N->getOriginalAlign(), MMOFlags, AAInfo);
5528 GetExpandedInteger(
N->getOperand(0), InL, InH);
5537 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5538 cast<AtomicSDNode>(
N)->getMemOperand());
5542SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5543 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5544 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5548 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5553SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5556 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5557 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5563SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
5566 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5567 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5576SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
5578 EVT OutVT =
N->getValueType(0);
5580 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5584 SDValue BaseIdx =
N->getOperand(1);
5611 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5620 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5624 "Promoted operand has an element type greater than result");
5637 InOp0 = GetPromotedInteger(
N->getOperand(0));
5644 for (
unsigned i = 0; i != OutNumElems; ++i) {
5660SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
5661 EVT OutVT =
N->getValueType(0);
5663 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5675 Vec = GetPromotedInteger(Vec);
5681SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
5684 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5690SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
5692 EVT VT =
N->getValueType(0);
5697 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5698 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5705 EVT OutVT =
N->getValueType(0);
5707 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5708 unsigned NumElems =
N->getNumOperands();
5716 for (
unsigned i = 0; i != NumElems; ++i) {
5718 EVT OpVT =
Op.getValueType();
5723 if (OpVT.
bitsLT(NOutVTElem)) {
5729 ExtOpc = NOutExtOpc;
5742 assert(!
N->getOperand(0).getValueType().isVector() &&
5743 "Input must be a scalar");
5745 EVT OutVT =
N->getValueType(0);
5747 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5752 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
5757 EVT OutVT =
N->getValueType(0);
5760 "Type must be promoted to a scalable vector type");
5761 const APInt &StepVal =
N->getConstantOperandAPInt(0);
5766SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
5769 EVT OutVT =
N->getValueType(0);
5771 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5773 unsigned NumOperands =
N->getNumOperands();
5778 SDUse *MaxSizedValue = std::max_element(
5780 EVT AVT = A.getValueType().getVectorElementType();
5781 EVT BVT = B.getValueType().getVectorElementType();
5782 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5788 for (
unsigned I = 0;
I < NumOperands; ++
I) {
5790 EVT OpVT =
Op.getValueType();
5792 Op = GetPromotedInteger(
Op);
5795 "Unhandled legalization type");
5812 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
5813 assert(NumElem * NumOperands == NumOutElem &&
5814 "Unexpected number of elements");
5818 for (
unsigned i = 0; i < NumOperands; ++i) {
5821 Op = GetPromotedInteger(
Op);
5822 EVT SclrTy =
Op.getValueType().getVectorElementType();
5823 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
5824 "Unexpected number of elements");
5826 for (
unsigned j = 0;
j < NumElem; ++
j) {
5836SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
5837 EVT VT =
N->getValueType(0);
5839 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
5847 if (getTypeAction(
N->getOperand(0).getValueType())
5851 switch(
N->getOpcode()) {
5853 Promoted = SExtPromotedInteger(
N->getOperand(0));
5856 Promoted = ZExtPromotedInteger(
N->getOperand(0));
5859 Promoted = GetPromotedInteger(
N->getOperand(0));
5864 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
5868 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
5871SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
5872 EVT OutVT =
N->getValueType(0);
5874 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5879 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5882 NOutVTElem,
N->getOperand(1));
5884 V0, ConvElem,
N->getOperand(2));
5892 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
5900 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
5901 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
5902 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
5905SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
5907 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5919SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
5924 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5934SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
5936 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5939 N->getValueType(0).getVectorNumElements());
5944SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
5947 EVT ResVT =
N->getValueType(0);
5948 unsigned NumElems =
N->getNumOperands();
5953 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5955 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
5963 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
5969 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5971 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
5972 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
5974 for (
unsigned i=0; i<NumElem; ++i) {
5986SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
5999 for (
unsigned I = 0;
I < OpNo;
I++)
6002 EVT Ty =
Op.getValueType();
6014 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6019 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6025SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6038 for (
unsigned I = 0;
I < OpNo;
I++)
6041 EVT Ty =
Op.getValueType();
6053 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6058 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)