29 #define DEBUG_TYPE "legalize-types"
39 void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
45 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
50 switch (
N->getOpcode()) {
53 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
54 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
66 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
68 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
70 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
72 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
73 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
74 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
76 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
82 Res = PromoteIntRes_Select(
N);
87 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
89 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
91 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
94 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
96 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
98 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(
N);
break;
100 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(
N);
break;
101 case ISD::VP_TRUNCATE:
103 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
104 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
108 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
110 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
112 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
114 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
116 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
118 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
120 Res = PromoteIntRes_BUILD_VECTOR(
N);
break;
122 Res = PromoteIntRes_SCALAR_TO_VECTOR(
N);
break;
124 Res = PromoteIntRes_SPLAT_VECTOR(
N);
break;
127 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
132 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
145 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
162 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
167 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
172 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
175 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
177 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
179 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
184 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(
N, ResNo);
break;
194 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(
N);
break;
206 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
209 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
223 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
227 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
239 Res = PromoteIntRes_VECREDUCE(
N);
242 case ISD::VP_REDUCE_ADD:
243 case ISD::VP_REDUCE_MUL:
244 case ISD::VP_REDUCE_AND:
245 case ISD::VP_REDUCE_OR:
246 case ISD::VP_REDUCE_XOR:
247 case ISD::VP_REDUCE_SMAX:
248 case ISD::VP_REDUCE_SMIN:
249 case ISD::VP_REDUCE_UMAX:
250 case ISD::VP_REDUCE_UMIN:
251 Res = PromoteIntRes_VP_REDUCE(
N);
255 Res = PromoteIntRes_FREEZE(
N);
260 Res = PromoteIntRes_Rotate(
N);
265 Res = PromoteIntRes_FunnelShift(
N);
269 Res = PromoteIntRes_IS_FPCLASS(
N);
275 SetPromotedInteger(
SDValue(
N, ResNo), Res);
280 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
281 return GetPromotedInteger(
Op);
286 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
288 Op.getValueType(),
Op,
N->getOperand(1));
293 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
295 Op.getValueType(),
Op,
N->getOperand(1));
301 N->getMemoryVT(), ResVT,
302 N->getChain(),
N->getBasePtr(),
311 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
314 N->getChain(),
N->getBasePtr(),
315 Op2,
N->getMemOperand());
326 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
337 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
347 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
350 Op2 = SExtPromotedInteger(Op2);
353 Op2 = ZExtPromotedInteger(Op2);
356 Op2 = GetPromotedInteger(Op2);
365 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
366 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
368 for (
unsigned i = 1, NumResults =
N->getNumValues();
i < NumResults; ++
i)
377 EVT OutVT =
N->getValueType(0);
381 switch (getTypeAction(InVT)) {
408 BitConvertToInteger(GetScalarizedVector(InOp)));
417 GetSplitVector(
N->getOperand(0), Lo, Hi);
418 Lo = BitConvertToInteger(Lo);
419 Hi = BitConvertToInteger(Hi);
427 JoinIntegers(Lo, Hi));
456 if (WidenInSize % OutSize == 0) {
457 unsigned Scale = WidenInSize / OutSize;
461 if (isTypeLegal(WideOutVT)) {
462 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
472 CreateStackStoreLoad(InOp, OutVT));
476 SDValue V = GetPromotedInteger(
N->getOperand(0));
482 SDValue Op = GetPromotedInteger(
N->getOperand(0));
483 EVT OVT =
N->getValueType(0);
484 EVT NVT =
Op.getValueType();
503 SDValue Op = GetPromotedInteger(
N->getOperand(0));
504 EVT OVT =
N->getValueType(0);
505 EVT NVT =
Op.getValueType();
529 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
534 EVT VT =
N->getValueType(0);
543 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
548 EVT OVT =
N->getValueType(0);
565 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
575 EVT OVT =
N->getValueType(0);
592 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
597 SDValue Op = GetPromotedInteger(
N->getOperand(0));
598 EVT OVT =
N->getValueType(0);
599 EVT NVT =
Op.getValueType();
625 return DAG.
getNode(
N->getOpcode(), dl, NVT,
Op);
628 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
643 EVT SVT =
In.getValueType().getScalarType();
655 unsigned NewOpc =
N->getOpcode();
673 if (
N->isStrictFPOpcode()) {
675 {
N->getOperand(0),
N->getOperand(1)});
680 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
695 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
699 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
707 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
715 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
727 if (getTypeAction(
N->getOperand(0).getValueType())
729 SDValue Res = GetPromotedInteger(
N->getOperand(0));
747 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
757 N->getMemoryVT(),
N->getMemOperand());
767 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
775 N->getOffset(),
N->getMask(), ExtPassThru,
776 N->getMemoryVT(),
N->getMemOperand(),
777 N->getAddressingMode(), ExtType,
778 N->isExpandingLoad());
787 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
789 "Gather result type and the passThru argument type should be the same");
796 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
797 N->getIndex(),
N->getScale() };
799 N->getMemoryVT(), dl, Ops,
800 N->getMemOperand(),
N->getIndexType(),
813 EVT VT =
N->getValueType(0);
814 EVT SVT = getSetCCResultType(VT);
815 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
816 unsigned NumOps =
N->getNumOperands();
817 assert(NumOps <= 3 &&
"Too many operands");
819 Ops[2] =
N->getOperand(2);
827 ReplaceValueWith(
SDValue(
N, 0), Res);
846 unsigned Opcode =
N->getOpcode();
849 SDValue Op1Promoted, Op2Promoted;
851 Op1Promoted = GetPromotedInteger(Op1);
852 Op2Promoted = ZExtPromotedInteger(Op2);
854 Op1Promoted = ZExtPromotedInteger(Op1);
855 Op2Promoted = ZExtPromotedInteger(Op2);
857 Op1Promoted = SExtPromotedInteger(Op1);
858 Op2Promoted = SExtPromotedInteger(Op2);
891 "addition, subtraction or left shift");
894 unsigned SHLAmount = NewBits - OldBits;
904 DAG.
getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
905 return DAG.
getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
914 DAG.
getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
923 SDValue Op1Promoted, Op2Promoted;
929 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
930 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
932 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
933 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
935 EVT OldType =
N->getOperand(0).getValueType();
949 Op2Promoted,
N->getOperand(2));
951 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
954 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
959 unsigned SatW,
bool Signed,
989 EVT VT =
LHS.getValueType();
1013 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1019 "Tried to saturate to more than the original type?");
1028 SDValue Op1Promoted, Op2Promoted;
1034 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1035 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1037 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1038 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1041 unsigned Scale =
N->getConstantOperandVal(2);
1050 N->getValueType(0).getScalarSizeInBits();
1055 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1056 Op2Promoted,
N->getOperand(2));
1066 Op2Promoted, Scale, DAG)) {
1069 N->getValueType(0).getScalarSizeInBits(),
1077 N->getValueType(0).getScalarSizeInBits());
1080 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1082 return PromoteIntRes_Overflow(
N);
1086 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1087 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1088 EVT OVT =
N->getOperand(0).getValueType();
1089 EVT NVT =
LHS.getValueType();
1104 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1112 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1113 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1115 unsigned Opcode =
N->getOpcode();
1116 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1123 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1124 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1126 LHS.getValueType(),
N->getOperand(0),
1127 N->getOperand(1),
LHS,
RHS,
N->getOperand(4));
1131 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1132 EVT InVT =
N->getOperand(OpNo).getValueType();
1135 EVT SVT = getSetCCResultType(InVT);
1143 SVT = getSetCCResultType(InVT);
1151 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1152 "Vector compare must return a vector result!");
1156 if (
N->isStrictFPOpcode()) {
1158 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1159 N->getOperand(2),
N->getOperand(3)};
1160 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers);
1165 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1166 N->getOperand(1),
N->getOperand(2));
1181 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1184 RHS = ZExtPromotedInteger(
RHS);
1185 if (
N->getOpcode() != ISD::VP_SHL)
1188 N->getOperand(2),
N->getOperand(3));
1191 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1192 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1194 Op.getValueType(),
Op,
N->getOperand(1));
1197 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1201 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1202 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1203 if (
N->getNumOperands() == 2)
1205 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1206 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1208 N->getOperand(2),
N->getOperand(3));
1211 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(
SDNode *
N) {
1213 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1214 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1215 if (
N->getNumOperands() == 2)
1217 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1218 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1220 N->getOperand(2),
N->getOperand(3));
1223 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(
SDNode *
N) {
1225 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1226 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1227 if (
N->getNumOperands() == 2)
1229 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1230 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1232 N->getOperand(2),
N->getOperand(3));
1238 SDValue LHS = SExtOrZExtPromotedInteger(
N->getOperand(0));
1239 SDValue RHS = SExtOrZExtPromotedInteger(
N->getOperand(1));
1246 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1249 RHS = ZExtPromotedInteger(
RHS);
1250 if (
N->getOpcode() != ISD::VP_ASHR)
1253 N->getOperand(2),
N->getOperand(3));
1258 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1261 RHS = ZExtPromotedInteger(
RHS);
1262 if (
N->getOpcode() != ISD::VP_LSHR)
1265 N->getOperand(2),
N->getOperand(3));
1271 ReplaceValueWith(
SDValue(
N, 0), Res);
1276 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1277 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1280 Amt = ZExtPromotedInteger(Amt);
1284 EVT OldVT =
N->getOperand(0).getValueType();
1285 EVT VT =
Lo.getValueType();
1286 unsigned Opcode =
N->getOpcode();
1300 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1321 return DAG.
getNode(Opcode,
DL, VT, Hi, Lo, Amt);
1337 Res = GetPromotedInteger(InOp);
1344 "Dst and Src must have the same number of elements");
1346 "Promoted vector type must be a power of two");
1349 GetSplitVector(InOp, EOp1, EOp2);
1357 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1358 "Expected VP_TRUNCATE opcode");
1359 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1360 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1361 std::tie(EVLLo, EVLHi) =
1362 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1363 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1364 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1371 SDValue WideInOp = GetWidenedVector(InOp);
1376 N->getValueType(0).getScalarType(), NumElem);
1391 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1392 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1397 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1399 return PromoteIntRes_Overflow(
N);
1403 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1404 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1405 EVT OVT =
N->getOperand(0).getValueType();
1406 EVT NVT =
LHS.getValueType();
1420 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1428 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(
SDNode *
N,
unsigned ResNo) {
1430 return PromoteIntRes_Overflow(
N);
1442 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1443 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1445 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1457 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(
SDNode *
N,
1459 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1460 return PromoteIntRes_Overflow(
N);
1464 EVT OVT =
N->getValueType(0);
1477 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1481 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1484 return PromoteIntRes_Overflow(
N);
1488 EVT SmallVT =
LHS.getValueType();
1495 LHS = SExtPromotedInteger(
LHS);
1496 RHS = SExtPromotedInteger(
RHS);
1498 LHS = ZExtPromotedInteger(
LHS);
1499 RHS = ZExtPromotedInteger(
RHS);
1514 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Hi,
1530 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1536 N->getValueType(0)));
1542 APInt MulImm = cast<ConstantSDNode>(
N->getOperand(0))->getAPIntValue();
1549 EVT VT =
N->getValueType(0);
1557 for (
unsigned i = 0;
i < NumRegs; ++
i) {
1558 Parts[
i] = DAG.
getVAArg(RegVT, dl, Chain, Ptr,
N->getOperand(2),
1559 N->getConstantOperandVal(3));
1570 for (
unsigned i = 1;
i < NumRegs; ++
i) {
1581 ReplaceValueWith(
SDValue(
N, 1), Chain);
1594 bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1598 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1603 switch (
N->getOpcode()) {
1606 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1607 N->dump(&DAG);
dbgs() <<
"\n";
1613 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1616 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1617 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1623 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
break;
1625 Res = PromoteIntOp_SCALAR_TO_VECTOR(
N);
break;
1627 Res = PromoteIntOp_SPLAT_VECTOR(
N);
break;
1629 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1632 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1636 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1638 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1640 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1642 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1644 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1646 case ISD::VP_TRUNCATE:
1659 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1662 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1667 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(
N, OpNo);
break;
1695 case ISD::VP_REDUCE_ADD:
1696 case ISD::VP_REDUCE_MUL:
1697 case ISD::VP_REDUCE_AND:
1698 case ISD::VP_REDUCE_OR:
1699 case ISD::VP_REDUCE_XOR:
1700 case ISD::VP_REDUCE_SMAX:
1701 case ISD::VP_REDUCE_SMIN:
1702 case ISD::VP_REDUCE_UMAX:
1703 case ISD::VP_REDUCE_UMIN:
1704 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
1711 if (!Res.getNode())
return false;
1715 if (Res.getNode() ==
N)
1718 const bool IsStrictFp =
N->isStrictFPOpcode();
1719 assert(Res.getValueType() ==
N->getValueType(0) &&
1720 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1721 "Invalid operand expansion");
1725 ReplaceValueWith(
SDValue(
N, 0), Res);
1743 LHS = SExtPromotedInteger(
LHS);
1744 RHS = SExtPromotedInteger(
RHS);
1749 "Unknown integer comparison!");
1758 unsigned OpLEffectiveBits =
1760 unsigned OpREffectiveBits =
1762 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1763 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1770 LHS = SExtPromotedInteger(
LHS);
1771 RHS = SExtPromotedInteger(
RHS);
1782 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1783 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1790 LHS = ZExtPromotedInteger(
LHS);
1791 RHS = ZExtPromotedInteger(
RHS);
1795 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1800 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
1802 N->getChain(),
N->getBasePtr(), Op2,
N->getMemOperand());
1808 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
1811 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
1812 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
1816 PromoteSetCCOperands(
LHS,
RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
1821 N->getOperand(1),
LHS,
RHS,
N->getOperand(4)),
1825 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
1826 assert(OpNo == 1 &&
"only know how to promote condition");
1833 N->getOperand(2)), 0);
1838 EVT OVT =
N->getOperand(0).getValueType();
1839 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
1840 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
1841 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
1854 EVT VecVT =
N->getValueType(0);
1857 "Legal vector of one illegal element?");
1862 assert(
N->getOperand(0).getValueSizeInBits() >=
1863 N->getValueType(0).getScalarSizeInBits() &&
1864 "Type of inserted value narrower than vector element type!");
1867 for (
unsigned i = 0;
i < NumElts; ++
i)
1868 NewOps.push_back(GetPromotedInteger(
N->getOperand(
i)));
1873 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
1880 assert(
N->getOperand(1).getValueSizeInBits() >=
1881 N->getValueType(0).getScalarSizeInBits() &&
1882 "Type of inserted value narrower than vector element type!");
1884 GetPromotedInteger(
N->getOperand(1)),
1889 assert(OpNo == 2 &&
"Different operand and result vector types?");
1895 N->getOperand(1), Idx), 0);
1898 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(
SDNode *
N) {
1902 GetPromotedInteger(
N->getOperand(0))), 0);
1912 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
1913 assert(OpNo == 0 &&
"Only know how to promote the condition!");
1915 EVT OpTy =
N->getOperand(1).getValueType();
1918 if (
SDValue Res = WidenVSELECTMask(
N))
1920 Res,
N->getOperand(1),
N->getOperand(2));
1924 Cond = PromoteTargetBoolean(
Cond, OpVT);
1927 N->getOperand(2)), 0);
1930 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
1931 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
1935 PromoteSetCCOperands(
LHS,
RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
1939 N->getOperand(3),
N->getOperand(4)), 0);
1942 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
1943 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
1947 PromoteSetCCOperands(
LHS,
RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
1950 if (
N->getNumOperands() == 3)
1953 assert(
N->getNumOperands() == 5 &&
"Unexpected number of operands!");
1954 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
1957 N->getOperand(3),
N->getOperand(4)),
1963 ZExtPromotedInteger(
N->getOperand(1))), 0);
1968 ZExtPromotedInteger(
N->getOperand(2))), 0);
1972 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1981 SExtPromotedInteger(
N->getOperand(0))), 0);
1984 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
1986 SExtPromotedInteger(
N->getOperand(1))), 0);
1991 SDValue Ch =
N->getChain(), Ptr =
N->getBasePtr();
1994 SDValue Val = GetPromotedInteger(
N->getValue());
1998 N->getMemoryVT(),
N->getMemOperand());
2009 Mask = PromoteTargetBoolean(
Mask, DataVT);
2015 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2016 DataOp = GetPromotedInteger(DataOp);
2019 N->getOffset(),
Mask,
N->getMemoryVT(),
2020 N->getMemOperand(),
N->getAddressingMode(),
2021 true,
N->isCompressingStore());
2026 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2027 EVT DataVT =
N->getValueType(0);
2028 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2030 NewOps[OpNo] =
Mask;
2047 EVT DataVT =
N->getValueType(0);
2048 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2049 }
else if (OpNo == 4) {
2051 if (
N->isIndexSigned())
2053 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2055 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2057 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2071 bool TruncateStore =
N->isTruncatingStore();
2076 EVT DataVT =
N->getValue().getValueType();
2077 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2078 }
else if (OpNo == 4) {
2080 if (
N->isIndexSigned())
2082 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2084 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2086 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2087 TruncateStore =
true;
2091 SDLoc(
N), NewOps,
N->getMemOperand(),
2092 N->getIndexType(), TruncateStore);
2096 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2097 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2099 N->getOperand(1),
N->getOperand(2));
2105 ZExtPromotedInteger(
N->getOperand(0))), 0);
2108 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2110 ZExtPromotedInteger(
N->getOperand(1))), 0);
2115 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2120 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(
SDNode *
N,
unsigned OpNo) {
2121 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2128 Carry = PromoteTargetBoolean(Carry,
LHS.getValueType());
2134 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2139 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2141 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2145 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(
SDNode *
N,
unsigned OpNo) {
2146 assert(OpNo > 1 &&
"Don't know how to promote this operand!");
2149 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2150 SDValue Op3 = ZExtPromotedInteger(
N->getOperand(3));
2151 SDValue Op4 = ZExtPromotedInteger(
N->getOperand(4));
2158 bool IsStrict =
N->isStrictFPOpcode();
2170 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
2175 return DAG.
getUNDEF(
N->getValueType(0));
2177 unsigned OpOffset = IsStrict ? 1 : 0;
2180 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2181 "POWI exponent should match with sizeof(int) when doing the libcall.");
2184 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2185 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2186 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2187 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2189 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2194 switch (
N->getOpcode()) {
2202 case ISD::VP_REDUCE_ADD:
2203 case ISD::VP_REDUCE_MUL:
2204 case ISD::VP_REDUCE_AND:
2205 case ISD::VP_REDUCE_OR:
2206 case ISD::VP_REDUCE_XOR:
2210 case ISD::VP_REDUCE_SMAX:
2211 case ISD::VP_REDUCE_SMIN:
2215 case ISD::VP_REDUCE_UMAX:
2216 case ISD::VP_REDUCE_UMIN:
2226 return GetPromotedInteger(V);
2228 return SExtPromotedInteger(V);
2230 return ZExtPromotedInteger(V);
2236 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2238 EVT EltVT =
Op.getValueType().getVectorElementType();
2239 EVT VT =
N->getValueType(0);
2250 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2257 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2261 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2263 Op = PromoteIntOpVectorReduction(
N,
Op);
2267 EVT VT =
N->getValueType(0);
2268 EVT EltVT =
Op.getValueType().getScalarType();
2283 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2295 void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2302 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2305 switch (
N->getOpcode()) {
2308 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2309 N->dump(&DAG);
dbgs() <<
"\n";
2318 case ISD::UNDEF: SplitRes_UNDEF(
N, Lo, Hi);
break;
2325 case ISD::VAARG: ExpandRes_VAARG(
N, Lo, Hi);
break;
2331 case ISD::BSWAP: ExpandIntRes_BSWAP(
N, Lo, Hi);
break;
2332 case ISD::PARITY: ExpandIntRes_PARITY(
N, Lo, Hi);
break;
2334 case ISD::ABS: ExpandIntRes_ABS(
N, Lo, Hi);
break;
2336 case ISD::CTLZ: ExpandIntRes_CTLZ(
N, Lo, Hi);
break;
2337 case ISD::CTPOP: ExpandIntRes_CTPOP(
N, Lo, Hi);
break;
2339 case ISD::CTTZ: ExpandIntRes_CTTZ(
N, Lo, Hi);
break;
2350 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(
N, Lo, Hi);
break;
2351 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N), Lo, Hi);
break;
2352 case ISD::MUL: ExpandIntRes_MUL(
N, Lo, Hi);
break;
2354 case ISD::SDIV: ExpandIntRes_SDIV(
N, Lo, Hi);
break;
2357 case ISD::SREM: ExpandIntRes_SREM(
N, Lo, Hi);
break;
2359 case ISD::UDIV: ExpandIntRes_UDIV(
N, Lo, Hi);
break;
2360 case ISD::UREM: ExpandIntRes_UREM(
N, Lo, Hi);
break;
2377 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2378 SplitInteger(Tmp.first, Lo, Hi);
2379 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2387 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2396 SplitInteger(Tmp, Lo, Hi);
2404 case ISD::XOR: ExpandIntRes_Logical(
N, Lo, Hi);
break;
2409 case ISD::SMIN: ExpandIntRes_MINMAX(
N, Lo, Hi);
break;
2412 case ISD::SUB: ExpandIntRes_ADDSUB(
N, Lo, Hi);
break;
2415 case ISD::SUBC: ExpandIntRes_ADDSUBC(
N, Lo, Hi);
break;
2418 case ISD::SUBE: ExpandIntRes_ADDSUBE(
N, Lo, Hi);
break;
2428 case ISD::SRL: ExpandIntRes_Shift(
N, Lo, Hi);
break;
2431 case ISD::SSUBO: ExpandIntRes_SADDSUBO(
N, Lo, Hi);
break;
2433 case ISD::USUBO: ExpandIntRes_UADDSUBO(
N, Lo, Hi);
break;
2435 case ISD::SMULO: ExpandIntRes_XMULO(
N, Lo, Hi);
break;
2440 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(
N, Lo, Hi);
break;
2467 ExpandIntRes_Rotate(
N, Lo, Hi);
2472 ExpandIntRes_FunnelShift(
N, Lo, Hi);
2476 ExpandIntRes_VSCALE(
N, Lo, Hi);
2482 SetExpandedInteger(
SDValue(
N, ResNo), Lo, Hi);
2486 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *Node) {
2487 unsigned Opc = Node->getOpcode();
2489 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2493 EVT RetVT = Node->getValueType(0);
2497 Ops.
append(Node->op_begin() + 2, Node->op_end());
2498 Ops.push_back(Node->getOperand(1));
2501 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2502 "Unexpected atomic op or value type!");
2503 Ops.
append(Node->op_begin() + 1, Node->op_end());
2506 Node->getOperand(0));
2511 void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2516 GetExpandedInteger(
N->getOperand(0), InL, InH);
2527 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2529 EVT ShTy =
N->getOperand(1).getValueType();
2532 if (Amt.
uge(VTBits)) {
2534 }
else if (Amt.
ugt(NVTBits)) {
2538 }
else if (Amt == NVTBits) {
2553 if (Amt.
uge(VTBits)) {
2555 }
else if (Amt.
ugt(NVTBits)) {
2559 }
else if (Amt == NVTBits) {
2574 if (Amt.
uge(VTBits)) {
2577 }
else if (Amt.
ugt(NVTBits)) {
2582 }
else if (Amt == NVTBits) {
2600 bool DAGTypeLegalizer::
2608 "Expanded integer type size not a power of two!");
2615 if (((Known.
Zero|Known.
One) & HighBitMask) == 0)
2620 GetExpandedInteger(
N->getOperand(0), InL, InH);
2629 switch (
N->getOpcode()) {
2657 switch (
N->getOpcode()) {
2674 Lo = DAG.
getNode(
N->getOpcode(), dl, NVT, InL, Amt);
2687 bool DAGTypeLegalizer::
2694 "Expanded integer type size not a power of two!");
2699 GetExpandedInteger(
N->getOperand(0), InL, InH);
2711 switch (
N->getOpcode()) {
2726 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
2742 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
2758 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
2779 void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
2787 SDValue LHSL, LHSH, RHSL, RHSH;
2788 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
2789 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
2793 EVT CCT = getSetCCResultType(NVT);
2796 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
2812 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
2816 SDValue LHSL, LHSH, RHSL, RHSH;
2817 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
2818 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
2821 SDValue LoOps[2] = { LHSL, RHSL };
2822 SDValue HiOps[3] = { LHSH, RHSH };
2831 HiOps[2] =
Lo.getValue(1);
2835 HiOps[2] =
Lo.getValue(1);
2855 HiOps[2] =
Lo.getValue(1);
2859 HiOps[2] =
Lo.getValue(1);
2872 EVT OvfVT = getSetCCResultType(NVT);
2892 Hi = DAG.
getNode(
N->getOpcode(), dl, NVT, Hi, OVF);
2896 Hi = DAG.
getNode(RevOpc, dl, NVT, Hi, OVF);
2904 SDValue Cmp1 = DAG.
getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2916 SDValue Cmp2 = DAG.
getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2925 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2939 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
2942 SDValue LHSL, LHSH, RHSL, RHSH;
2944 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
2945 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
2947 SDValue LoOps[2] = { LHSL, RHSL };
2948 SDValue HiOps[3] = { LHSH, RHSH };
2952 HiOps[2] =
Lo.getValue(1);
2956 HiOps[2] =
Lo.getValue(1);
2962 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
2965 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
2968 SDValue LHSL, LHSH, RHSL, RHSH;
2970 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
2971 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
2973 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
2974 SDValue HiOps[3] = { LHSH, RHSH };
2976 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
2977 HiOps[2] =
Lo.getValue(1);
2978 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
2982 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
2985 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
2993 unsigned CarryOp, NoCarryOp;
2995 switch(
N->getOpcode()) {
3015 SDValue LHSL, LHSH, RHSL, RHSH;
3016 GetExpandedInteger(
LHS, LHSL, LHSH);
3017 GetExpandedInteger(
RHS, RHSL, RHSH);
3019 SDValue LoOps[2] = { LHSL, RHSL };
3020 SDValue HiOps[3] = { LHSH, RHSH };
3022 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3023 HiOps[2] =
Lo.getValue(1);
3024 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3026 Ovf =
Hi.getValue(1);
3031 SplitInteger(Sum, Lo, Hi);
3040 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3043 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(
SDNode *
N,
3046 SDValue LHSL, LHSH, RHSL, RHSH;
3048 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3049 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3051 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3054 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3055 HiOps[2] =
Lo.getValue(1);
3056 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3060 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3063 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3066 SDValue LHSL, LHSH, RHSL, RHSH;
3068 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3069 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3075 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3076 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3080 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3083 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3088 if (
Op.getValueType().bitsLE(NVT)) {
3095 assert(getTypeAction(
Op.getValueType()) ==
3097 "Only know how to promote this result!");
3100 "Operand over promoted?");
3102 SplitInteger(Res, Lo, Hi);
3106 void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3109 GetExpandedInteger(
N->getOperand(0), Lo, Hi);
3110 EVT NVT =
Lo.getValueType();
3111 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3115 if (NVTBits < EVTBits) {
3118 EVTBits - NVTBits)));
3128 void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3131 GetExpandedInteger(
N->getOperand(0), Lo, Hi);
3132 EVT NVT =
Lo.getValueType();
3133 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3137 if (NVTBits < EVTBits) {
3140 EVTBits - NVTBits)));
3148 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3151 GetExpandedInteger(
N->getOperand(0), Hi, Lo);
3156 void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3159 GetExpandedInteger(
N->getOperand(0), Hi, Lo);
3164 void DAGTypeLegalizer::ExpandIntRes_PARITY(
SDNode *
N,
SDValue &Lo,
3168 GetExpandedInteger(
N->getOperand(0), Lo, Hi);
3169 EVT NVT =
Lo.getValueType();
3175 void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3179 auto Constant = cast<ConstantSDNode>(
N);
3181 bool IsTarget =
Constant->isTargetOpcode();
3182 bool IsOpaque =
Constant->isOpaque();
3193 GetExpandedInteger(N0, Lo, Hi);
3194 EVT NVT =
Lo.getValueType();
3217 EVT VT =
N->getValueType(0);
3221 SplitInteger(Neg, NegLo, NegHi);
3229 void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3233 GetExpandedInteger(
N->getOperand(0), Lo, Hi);
3234 EVT NVT =
Lo.getValueType();
3249 void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3253 GetExpandedInteger(
N->getOperand(0), Lo, Hi);
3254 EVT NVT =
Lo.getValueType();
3260 void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3264 GetExpandedInteger(
N->getOperand(0), Lo, Hi);
3265 EVT NVT =
Lo.getValueType();
3280 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(
SDNode *
N,
SDValue &Lo,
3294 ReplaceValueWith(
SDValue(
N, 1), Chain);
3297 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(
SDNode *
N,
SDValue &Lo,
3300 EVT VT =
N->getValueType(0);
3302 bool IsStrict =
N->isStrictFPOpcode();
3304 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3306 Op = GetPromotedFloat(
Op);
3310 Op = GetSoftPromotedHalf(
Op);
3313 SplitInteger(
Op, Lo, Hi);
3318 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-sint conversion!");
3321 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3322 CallOptions, dl, Chain);
3323 SplitInteger(Tmp.first, Lo, Hi);
3326 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3329 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(
SDNode *
N,
SDValue &Lo,
3332 EVT VT =
N->getValueType(0);
3334 bool IsStrict =
N->isStrictFPOpcode();
3336 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3338 Op = GetPromotedFloat(
Op);
3342 Op = GetSoftPromotedHalf(
Op);
3345 SplitInteger(
Op, Lo, Hi);
3350 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-uint conversion!");
3352 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3353 CallOptions, dl, Chain);
3354 SplitInteger(Tmp.first, Lo, Hi);
3357 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3360 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(
SDNode *
N,
SDValue &Lo,
3363 SplitInteger(Res, Lo, Hi);
3366 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(
SDNode *
N,
SDValue &Lo,
3368 SDValue Op =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0);
3371 "Input type needs to be promoted!");
3373 EVT VT =
Op.getValueType();
3379 LC = RTLIB::LLROUND_F32;
3381 LC = RTLIB::LLROUND_F64;
3383 LC = RTLIB::LLROUND_F80;
3385 LC = RTLIB::LLROUND_F128;
3387 LC = RTLIB::LLROUND_PPCF128;
3388 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3392 LC = RTLIB::LLRINT_F32;
3394 LC = RTLIB::LLRINT_F64;
3396 LC = RTLIB::LLRINT_F80;
3398 LC = RTLIB::LLRINT_F128;
3400 LC = RTLIB::LLRINT_PPCF128;
3401 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
3406 EVT RetVT =
N->getValueType(0);
3411 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
3412 Op, CallOptions, dl,
3414 SplitInteger(Tmp.first, Lo, Hi);
3416 if (
N->isStrictFPOpcode())
3417 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3420 void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
3422 if (
N->isAtomic()) {
3425 EVT VT =
N->getMemoryVT();
3430 VT, VTs,
N->getOperand(0),
3431 N->getOperand(1), Zero, Zero,
N->getMemOperand());
3438 ExpandRes_NormalLoad(
N, Lo, Hi);
3444 EVT VT =
N->getValueType(0);
3455 if (
N->getMemoryVT().bitsLE(NVT)) {
3456 EVT MemVT =
N->getMemoryVT();
3458 Lo = DAG.
getExtLoad(ExtType, dl, NVT, Ch, Ptr,
N->getPointerInfo(), MemVT,
3459 N->getOriginalAlign(), MMOFlags, AAInfo);
3462 Ch =
Lo.getValue(1);
3467 unsigned LoSize =
Lo.getValueSizeInBits();
3481 Lo = DAG.
getLoad(NVT, dl, Ch, Ptr,
N->getPointerInfo(),
3482 N->getOriginalAlign(), MMOFlags, AAInfo);
3484 unsigned ExcessBits =
3492 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3493 N->getOriginalAlign(), MMOFlags, AAInfo);
3502 EVT MemVT =
N->getMemoryVT();
3505 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3508 Hi = DAG.
getExtLoad(ExtType, dl, NVT, Ch, Ptr,
N->getPointerInfo(),
3511 N->getOriginalAlign(), MMOFlags, AAInfo);
3517 N->getPointerInfo().getWithOffset(IncrementSize),
3519 N->getOriginalAlign(), MMOFlags, AAInfo);
3543 ReplaceValueWith(
SDValue(
N, 1), Ch);
3546 void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
3550 GetExpandedInteger(
N->getOperand(0), LL, LH);
3551 GetExpandedInteger(
N->getOperand(1), RL, RH);
3556 void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
3558 EVT VT =
N->getValueType(0);
3563 GetExpandedInteger(
N->getOperand(0), LL, LH);
3564 GetExpandedInteger(
N->getOperand(1), RL, RH);
3574 LC = RTLIB::MUL_I16;
3576 LC = RTLIB::MUL_I32;
3578 LC = RTLIB::MUL_I64;
3580 LC = RTLIB::MUL_I128;
3588 unsigned HalfBits =
Bits >> 1;
3624 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
3627 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3631 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(
SDNode *
N,
SDValue &Lo,
3639 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
3642 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(
SDNode *
N,
SDValue &Lo,
3645 SplitInteger(Result, Lo, Hi);
3648 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(
SDNode *
N,
SDValue &Lo,
3651 SplitInteger(Result, Lo, Hi);
3659 void DAGTypeLegalizer::ExpandIntRes_MULFIX(
SDNode *
N,
SDValue &Lo,
3662 EVT VT =
N->getValueType(0);
3666 uint64_t Scale =
N->getConstantOperandVal(2);
3678 EVT BoolVT = getSetCCResultType(VT);
3703 SplitInteger(Result, Lo, Hi);
3709 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
3713 GetExpandedInteger(
LHS, LL, LH);
3714 GetExpandedInteger(
RHS, RL, RH);
3726 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
3727 "the size of the current value type");
3749 if (Scale % NVTSize) {
3765 if (Scale == VTSize)
3786 EVT BoolNVT = getSetCCResultType(NVT);
3789 if (Scale < NVTSize) {
3796 }
else if (Scale == NVTSize) {
3799 }
else if (Scale < VTSize) {
3807 "(and saturation can't happen with Scale==VTSize).");
3814 if (Scale < NVTSize) {
3819 unsigned OverflowBits = VTSize - Scale + 1;
3820 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3821 "Extent of overflow bits must start within HL");
3838 }
else if (Scale == NVTSize) {
3851 }
else if (Scale < VTSize) {
3854 unsigned OverflowBits = VTSize - Scale + 1;
3875 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(
SDNode *
N,
SDValue &Lo,
3881 N->getConstantOperandVal(2), DAG);
3885 N->getConstantOperandVal(2), TLI, DAG);
3886 SplitInteger(Res, Lo, Hi);
3889 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *Node,
3892 "Node has unexpected Opcode");
3899 bool IsAdd = Node->getOpcode() ==
ISD::SADDO;
3907 SDValue LHSL, LHSH, RHSL, RHSH;
3908 GetExpandedInteger(
LHS, LHSL, LHSH);
3909 GetExpandedInteger(
RHS, RHSL, RHSH);
3913 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
3915 Ovf =
Hi.getValue(1);
3922 SplitInteger(Sum, Lo, Hi);
3947 EVT VT =
LHS.getValueType();
3950 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
3954 EVT OType = Node->getValueType(1);
3959 ReplaceValueWith(
SDValue(Node, 1), Ovf);
3974 TargetLowering::ArgListEntry Entry;
3979 EVT ArgVT =
N->op_begin()->getValueType();
3981 "Unexpected argument type for lowering");
3985 Entry.Node = Output;
3987 Entry.IsSExt =
false;
3988 Entry.IsZExt =
false;
3989 Args.push_back(Entry);
3994 Entry.Node = StackPtr;
3996 Entry.IsSExt =
false;
3997 Entry.IsZExt =
false;
3998 Args.push_back(Entry);
4001 int Bits =
N->getOperand(0)
4004 ->getIntegerBitWidth();
4007 Entry.IsSExt =
false;
4008 Entry.IsZExt =
true;
4009 Args.push_back(Entry);
4026 void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4028 EVT VT =
N->getValueType(0);
4030 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4034 SplitInteger(Res.
getValue(0), Lo, Hi);
4040 LC = RTLIB::SDIV_I16;
4042 LC = RTLIB::SDIV_I32;
4044 LC = RTLIB::SDIV_I64;
4046 LC = RTLIB::SDIV_I128;
4051 SplitInteger(Result, Lo, Hi);
4055 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4059 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4062 void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4064 EVT VT =
N->getValueType(0);
4070 return ExpandShiftByConstant(
N, CN->getAPIntValue(), Lo, Hi);
4074 if (ExpandShiftWithKnownAmountBit(
N, Lo, Hi))
4081 }
else if (
N->getOpcode() ==
ISD::SRL) {
4093 const bool LegalOrCustom =
4100 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4106 SDValue ShiftOp =
N->getOperand(1);
4111 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4113 Hi =
Lo.getValue(1);
4123 LC = RTLIB::SHL_I16;
4125 LC = RTLIB::SHL_I32;
4127 LC = RTLIB::SHL_I64;
4129 LC = RTLIB::SHL_I128;
4130 }
else if (
N->getOpcode() ==
ISD::SRL) {
4133 LC = RTLIB::SRL_I16;
4135 LC = RTLIB::SRL_I32;
4137 LC = RTLIB::SRL_I64;
4139 LC = RTLIB::SRL_I128;
4144 LC = RTLIB::SRA_I16;
4146 LC = RTLIB::SRA_I32;
4148 LC = RTLIB::SRA_I64;
4150 LC = RTLIB::SRA_I128;
4157 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4159 CallOptions.
setSExt(isSigned);
4160 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4164 if (!ExpandShiftWithUnknownAmountBit(
N, Lo, Hi))
4168 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4173 if (
Op.getValueType().bitsLE(NVT)) {
4184 assert(getTypeAction(
Op.getValueType()) ==
4186 "Only know how to promote this result!");
4189 "Operand over promoted?");
4191 SplitInteger(Res, Lo, Hi);
4199 void DAGTypeLegalizer::
4202 GetExpandedInteger(
N->getOperand(0), Lo, Hi);
4203 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4225 void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4227 EVT VT =
N->getValueType(0);
4229 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4233 SplitInteger(Res.
getValue(1), Lo, Hi);
4239 LC = RTLIB::SREM_I16;
4241 LC = RTLIB::SREM_I32;
4243 LC = RTLIB::SREM_I64;
4245 LC = RTLIB::SREM_I128;
4250 SplitInteger(Result, Lo, Hi);
4254 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4258 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4261 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4273 void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,