27#define DEBUG_TYPE "legalize-types"
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
51void DAGTypeLegalizer::SoftenFloatResult(
SDNode *
N,
unsigned ResNo) {
52 LLVM_DEBUG(
dbgs() <<
"Soften float result " << ResNo <<
": ";
N->dump(&DAG));
55 switch (
N->getOpcode()) {
59 dbgs() <<
"SoftenFloatResult #" << ResNo <<
": ";
60 N->dump(&DAG);
dbgs() <<
"\n";
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(
N, ResNo);
break;
115 case ISD::FMA:
R = SoftenFloatRes_FMA(
N);
break;
186 SetSoftenedFloat(
SDValue(
N, ResNo), R);
190SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(
SDNode *
N, RTLIB::Libcall LC) {
191 bool IsStrict =
N->isStrictFPOpcode();
193 unsigned Offset = IsStrict ? 1 : 0;
195 "Unexpected number of operands!");
199 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
201 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
205 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
209SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(
SDNode *
N, RTLIB::Libcall LC) {
210 bool IsStrict =
N->isStrictFPOpcode();
212 unsigned Offset = IsStrict ? 1 : 0;
214 "Unexpected number of operands!");
216 GetSoftenedFloat(
N->getOperand(1 +
Offset)) };
219 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
220 N->getOperand(1 +
Offset).getValueType() };
222 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Ops,
226 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
231 return BitConvertToInteger(
N->getOperand(0));
237 GetSoftenedFloat(
N->getOperand(0)));
243 GetSoftenedFloat(
N->getOperand(0)));
249 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
250 return BitConvertToInteger(
Op);
258 BitConvertToInteger(
N->getOperand(0)),
259 BitConvertToInteger(
N->getOperand(1)));
275 APInt Val(128, words);
286SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(
SDNode *
N) {
288 assert(Src.getValueType() == MVT::ppcf128 &&
289 "In floats only ppcf128 can be extracted by element!");
291 N->getValueType(0).changeTypeToInteger(),
295SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N,
unsigned ResNo) {
296 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
299 NewOp,
N->getOperand(1));
310 SDValue Op = GetSoftenedFloat(
N->getOperand(0));
316 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
322 RTLIB::FMIN_PPCF128));
327 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
333 RTLIB::FMAX_PPCF128));
337 return SoftenFloatRes_Binary(
339 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
340 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128));
344 return SoftenFloatRes_Binary(
346 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
347 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128));
351 return SoftenFloatRes_Binary(
353 RTLIB::FMINIMUM_F64, RTLIB::FMINIMUM_F80,
354 RTLIB::FMINIMUM_F128, RTLIB::FMINIMUM_PPCF128));
358 return SoftenFloatRes_Binary(
360 RTLIB::FMAXIMUM_F64, RTLIB::FMAXIMUM_F80,
361 RTLIB::FMAXIMUM_F128, RTLIB::FMAXIMUM_PPCF128));
370 RTLIB::ADD_PPCF128));
374 return SoftenFloatRes_Unary(
375 N,
GetFPLibCall(
N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64,
376 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128));
380 return SoftenFloatRes_Unary(
381 N,
GetFPLibCall(
N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64,
382 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128));
386 return SoftenFloatRes_Unary(
387 N,
GetFPLibCall(
N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64,
388 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128));
392 return SoftenFloatRes_Binary(
394 GetFPLibCall(
N->getValueType(0), RTLIB::ATAN2_F32, RTLIB::ATAN2_F64,
395 RTLIB::ATAN2_F80, RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128));
404 RTLIB::CBRT_PPCF128));
413 RTLIB::CEIL_PPCF128));
418 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
421 EVT LVT =
LHS.getValueType();
422 EVT RVT =
RHS.getValueType();
443 }
else if (SizeDiff < 0) {
470 RTLIB::COS_PPCF128));
474 return SoftenFloatRes_Unary(
475 N,
GetFPLibCall(
N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64,
476 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128));
485 RTLIB::DIV_PPCF128));
494 RTLIB::EXP_PPCF128));
503 RTLIB::EXP2_PPCF128));
507 return SoftenFloatRes_Unary(
509 GetFPLibCall(
N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
510 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
519 RTLIB::FLOOR_PPCF128));
528 RTLIB::LOG_PPCF128));
537 RTLIB::LOG2_PPCF128));
546 RTLIB::LOG10_PPCF128));
550 bool IsStrict =
N->isStrictFPOpcode();
552 unsigned Offset = IsStrict ? 1 : 0;
554 GetSoftenedFloat(
N->getOperand(1 +
Offset)),
555 GetSoftenedFloat(
N->getOperand(2 +
Offset)) };
558 EVT OpsVT[3] = {
N->getOperand(0 +
Offset).getValueType(),
559 N->getOperand(1 +
Offset).getValueType(),
560 N->getOperand(2 +
Offset).getValueType() };
562 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG,
569 NVT, Ops, CallOptions,
SDLoc(
N), Chain);
571 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
581 RTLIB::MUL_PPCF128));
586 RTLIB::NEARBYINT_F32,
587 RTLIB::NEARBYINT_F64,
588 RTLIB::NEARBYINT_F80,
589 RTLIB::NEARBYINT_F128,
590 RTLIB::NEARBYINT_PPCF128));
604 bool IsStrict =
N->isStrictFPOpcode();
611 Op = GetPromotedFloat(
Op);
614 if (
Op.getValueType() ==
N->getValueType(0)) {
616 ReplaceValueWith(
SDValue(
N, 1), Chain);
617 return BitConvertToInteger(
Op);
625 if ((
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16) &&
626 N->getValueType(0) != MVT::f32) {
629 { MVT::f32, MVT::Other }, { Chain,
Op });
630 Chain =
Op.getValue(1);
636 if (
Op.getValueType() == MVT::bf16) {
638 return SoftenFloatRes_BF16_TO_FP(
N);
642 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
644 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
646 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
650 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
660 EVT OpsVT[1] = {
N->getOperand(0).getValueType() };
663 CallOptions,
SDLoc(
N)).first;
664 if (
N->getValueType(0) == MVT::f32)
669 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
676 assert(
N->getValueType(0) == MVT::f32 &&
677 "Can only soften BF16_TO_FP with f32 result");
689 bool IsStrict =
N->isStrictFPOpcode();
694 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND!");
696 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
698 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
702 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
711 bool IsStrict =
N->isStrictFPOpcode();
712 unsigned Offset = IsStrict ? 1 : 0;
719 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
725 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
730 N->getOperand(1 +
Offset).getValueType().getSizeInBits()) {
735 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
743 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
744 N->getOperand(1 +
Offset).getValueType() };
746 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Ops,
750 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
755 assert(!
N->isStrictFPOpcode() &&
"strictfp not implemented for frexp");
756 EVT VT0 =
N->getValueType(0);
757 EVT VT1 =
N->getValueType(1);
768 ReplaceValueWith(
SDValue(
N, 1), PoisonExp);
776 SDValue Ops[2] = {GetSoftenedFloat(
N->getOperand(0)), StackSlot};
783 .setOpsTypeOverrides(CallOpsTypeOverrides);
785 auto [ReturnVal, Chain] = TLI.
makeLibCall(DAG, LC, NVT0, Ops, CallOptions,
DL,
787 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
793 ReplaceValueWith(
SDValue(
N, 1), LoadExp);
797bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
798 SDNode *
N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
799 assert(!
N->isStrictFPOpcode() &&
"strictfp not implemented");
800 EVT VT =
N->getValueType(0);
802 assert(VT ==
N->getValueType(1) &&
803 "expected both return values to have the same type");
815 std::array<SDValue, 2> StackSlots;
818 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ++ResNum) {
819 if (ResNum == CallRetResNo)
824 StackSlots[ResNum] = StackSlot;
832 .setOpsTypeOverrides(CallOpsTypeOverrides);
834 auto [ReturnVal, Chain] = TLI.
makeLibCall(DAG, LC, NVT, Ops, CallOptions,
DL,
837 auto CreateStackLoad = [&, Chain = Chain](
SDValue StackSlot) {
838 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
841 return DAG.
getLoad(NVT,
DL, Chain, StackSlot, PtrInfo);
844 for (
auto [ResNum, SlackSlot] :
enumerate(StackSlots)) {
845 if (CallRetResNo == ResNum) {
846 SetSoftenedFloat(
SDValue(
N, ResNum), ReturnVal);
849 SetSoftenedFloat(
SDValue(
N, ResNum), CreateStackLoad(SlackSlot));
856 EVT VT =
N->getValueType(0);
871 SoftSin = SoftenFloatRes_Unary(
N, SinLC);
872 SoftCos = SoftenFloatRes_Unary(
N, CosLC);
875 SetSoftenedFloat(
SDValue(
N, 0), SoftSin);
876 SetSoftenedFloat(
SDValue(
N, 1), SoftCos);
881 EVT VT =
N->getValueType(0);
900 RTLIB::REM_PPCF128));
909 RTLIB::RINT_PPCF128));
918 RTLIB::ROUND_PPCF128));
923 RTLIB::ROUNDEVEN_F32,
924 RTLIB::ROUNDEVEN_F64,
925 RTLIB::ROUNDEVEN_F80,
926 RTLIB::ROUNDEVEN_F128,
927 RTLIB::ROUNDEVEN_PPCF128));
936 RTLIB::SIN_PPCF128));
940 return SoftenFloatRes_Unary(
941 N,
GetFPLibCall(
N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64,
942 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128));
951 RTLIB::SQRT_PPCF128));
960 RTLIB::SUB_PPCF128));
964 return SoftenFloatRes_Unary(
965 N,
GetFPLibCall(
N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
966 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
970 return SoftenFloatRes_Unary(
971 N,
GetFPLibCall(
N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64,
972 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128));
981 RTLIB::TRUNC_PPCF128));
986 EVT VT =
N->getValueType(0);
991 L->getMemOperand()->getFlags() &
995 NewL = DAG.
getLoad(
L->getAddressingMode(),
L->getExtensionType(), NVT, dl,
996 L->getChain(),
L->getBasePtr(),
L->getOffset(),
997 L->getPointerInfo(), NVT,
L->getBaseAlign(), MMOFlags,
1007 dl,
L->getChain(),
L->getBasePtr(),
L->getOffset(),
1008 L->getPointerInfo(),
L->getMemoryVT(),
L->getBaseAlign(),
1009 MMOFlags,
L->getAAInfo());
1014 return BitConvertToInteger(ExtendNode);
1019 EVT VT =
N->getValueType(0);
1026 {L->getChain(), L->getBasePtr()},
L->getMemOperand());
1041 LHS.getValueType(),
N->getOperand(0), LHS, RHS);
1048 LHS.getValueType(),
N->getOperand(0),
1049 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1054 N->getValueType(0)));
1060 EVT VT =
N->getValueType(0);
1065 NewVAARG = DAG.
getVAArg(NVT, dl, Chain,
Ptr,
N->getOperand(2),
1066 N->getConstantOperandVal(3));
1076 bool IsStrict =
N->isStrictFPOpcode();
1079 EVT SVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
1080 EVT RVT =
N->getValueType(0);
1087 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1088 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1089 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1095 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
1100 NVT,
N->getOperand(IsStrict ? 1 : 0));
1104 std::pair<SDValue, SDValue> Tmp =
1106 Op, CallOptions, dl, Chain);
1109 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1119SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
1128bool DAGTypeLegalizer::SoftenFloatOperand(
SDNode *
N,
unsigned OpNo) {
1129 LLVM_DEBUG(
dbgs() <<
"Soften float operand " << OpNo <<
": ";
N->dump(&DAG));
1132 switch (
N->getOpcode()) {
1135 dbgs() <<
"SoftenFloatOperand Op #" << OpNo <<
": ";
1136 N->dump(&DAG);
dbgs() <<
"\n";
1141 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(
N);
break;
1154 Res = SoftenFloatOp_FP_TO_XINT_SAT(
N);
break;
1156 case ISD::LROUND: Res = SoftenFloatOp_LROUND(
N);
break;
1160 case ISD::LRINT: Res = SoftenFloatOp_LRINT(
N);
break;
1162 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(
N);
break;
1166 case ISD::SETCC: Res = SoftenFloatOp_SETCC(
N);
break;
1167 case ISD::STORE: Res = SoftenFloatOp_STORE(
N, OpNo);
break;
1169 Res = SoftenFloatOp_ATOMIC_STORE(
N, OpNo);
1173 Res = SoftenFloatOp_FAKE_USE(
N);
1178 if (!Res.
getNode())
return false;
1186 "Invalid operand softening");
1188 ReplaceValueWith(
SDValue(
N, 0), Res);
1193 SDValue Op0 = GetSoftenedFloat(
N->getOperand(0));
1207 bool IsStrict =
N->isStrictFPOpcode();
1208 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
1209 EVT SVT =
Op.getValueType();
1210 EVT RVT =
N->getValueType(0);
1214 FloatRVT = MVT::f16;
1217 FloatRVT = MVT::bf16;
1220 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND libcall");
1223 Op = GetSoftenedFloat(
Op);
1226 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RVT,
Op,
1230 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1231 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1238 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
1239 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
1242 NewLHS = GetSoftenedFloat(NewLHS);
1243 NewRHS = GetSoftenedFloat(NewRHS);
1245 N->getOperand(2),
N->getOperand(3));
1249 if (!NewRHS.getNode()) {
1266 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1267 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1268 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1272 if (Promoted.
bitsGE(RetVT))
1280 bool IsStrict =
N->isStrictFPOpcode();
1284 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
1285 EVT SVT =
Op.getValueType();
1286 EVT RVT =
N->getValueType(0);
1296 "Unsupported FP_TO_XINT!");
1298 Op = GetSoftenedFloat(
Op);
1302 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
1303 CallOptions, dl, Chain);
1311 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1312 ReplaceValueWith(
SDValue(
N, 0), Res);
1316SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(
SDNode *
N) {
1322 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
1323 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
1326 NewLHS = GetSoftenedFloat(NewLHS);
1327 NewRHS = GetSoftenedFloat(NewRHS);
1329 N->getOperand(0),
N->getOperand(1));
1333 if (!NewRHS.getNode()) {
1340 N->getOperand(2),
N->getOperand(3),
1346 bool IsStrict =
N->isStrictFPOpcode();
1347 SDValue Op0 =
N->getOperand(IsStrict ? 1 : 0);
1348 SDValue Op1 =
N->getOperand(IsStrict ? 2 : 1);
1351 cast<CondCodeSDNode>(
N->getOperand(IsStrict ? 3 : 2))->get();
1354 SDValue NewLHS = GetSoftenedFloat(Op0);
1355 SDValue NewRHS = GetSoftenedFloat(Op1);
1371 "Unexpected setcc expansion!");
1374 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
1375 ReplaceValueWith(
SDValue(
N, 1), Chain);
1381SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
1383 assert(OpNo == 1 &&
"Can only soften the stored value!");
1388 if (
ST->isTruncatingStore())
1390 Val = BitConvertToInteger(
1394 Val = GetSoftenedFloat(Val);
1396 return DAG.
getStore(
ST->getChain(), dl, Val,
ST->getBasePtr(),
1397 ST->getMemOperand());
1400SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(
SDNode *
N,
unsigned OpNo) {
1401 assert(OpNo == 1 &&
"Can only soften the stored value!");
1407 assert(
ST->getMemoryVT() == VT &&
"truncating atomic store not handled");
1409 SDValue NewVal = GetSoftenedFloat(Val);
1411 ST->getBasePtr(),
ST->getMemOperand());
1416 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
1419 EVT LVT =
LHS.getValueType();
1421 EVT RVT =
RHS.getValueType();
1427 int SizeDiff = RSize - LSize;
1435 }
else if (SizeDiff < 0) {
1448SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(
SDNode *
N, RTLIB::Libcall LC) {
1450 bool IsStrict =
N->isStrictFPOpcode();
1451 unsigned Offset = IsStrict ? 1 : 0;
1455 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
1457 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
1461 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1462 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1470 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1476 RTLIB::LROUND_PPCF128));
1480 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1485 RTLIB::LLROUND_F128,
1486 RTLIB::LLROUND_PPCF128));
1490 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1496 RTLIB::LRINT_PPCF128));
1500 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1506 RTLIB::LLRINT_PPCF128));
1510 SDValue Op1 = BitConvertToInteger(
N->getOperand(1));
1512 N->getOperand(0), Op1);
1523void DAGTypeLegalizer::ExpandFloatResult(
SDNode *
N,
unsigned ResNo) {
1529 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
1532 switch (
N->getOpcode()) {
1535 dbgs() <<
"ExpandFloatResult #" << ResNo <<
": ";
1536 N->dump(&DAG);
dbgs() <<
"\n";
1653 "Do not know how to expand this float constant!");
1654 APInt C = cast<ConstantFPSDNode>(
N)->getValueAPF().bitcastToAPInt();
1661void DAGTypeLegalizer::ExpandFloatRes_Unary(
SDNode *
N, RTLIB::Libcall LC,
1663 bool IsStrict =
N->isStrictFPOpcode();
1664 unsigned Offset = IsStrict ? 1 : 0;
1668 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC,
N->getValueType(0),
1672 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1673 GetPairElements(Tmp.first,
Lo,
Hi);
1676void DAGTypeLegalizer::ExpandFloatRes_Binary(
SDNode *
N, RTLIB::Libcall LC,
1678 bool IsStrict =
N->isStrictFPOpcode();
1679 unsigned Offset = IsStrict ? 1 : 0;
1683 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC,
N->getValueType(0),
1684 Ops, CallOptions,
SDLoc(
N),
1687 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1688 GetPairElements(Tmp.first,
Lo,
Hi);
1691void DAGTypeLegalizer::ExpandFloatRes_FMODF(
SDNode *
N) {
1692 ExpandFloatRes_UnaryWithTwoFPResults(
N,
RTLIB::getMODF(
N->getValueType(0)),
1696void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(
SDNode *
N) {
1700void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(
SDNode *
N) {
1701 ExpandFloatRes_UnaryWithTwoFPResults(
N,
1705void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1706 SDNode *
N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1707 assert(!
N->isStrictFPOpcode() &&
"strictfp not implemented");
1712 GetPairElements(Res,
Lo,
Hi);
1719 assert(
N->getValueType(0) == MVT::ppcf128 &&
1720 "Logic only correct for ppcf128!");
1723 GetExpandedFloat(
N->getOperand(0),
Lo, Tmp);
1734 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1735 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1736 RTLIB::FMIN_PPCF128),
Lo,
Hi);
1742 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1743 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1744 RTLIB::FMAX_PPCF128),
Lo,
Hi);
1749 ExpandFloatRes_Binary(
1752 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
1753 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128),
1759 ExpandFloatRes_Binary(
1762 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
1763 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128),
1770 RTLIB::ADD_F32, RTLIB::ADD_F64,
1771 RTLIB::ADD_F80, RTLIB::ADD_F128,
1772 RTLIB::ADD_PPCF128),
Lo,
Hi);
1777 ExpandFloatRes_Unary(
N,
1779 RTLIB::ACOS_F64, RTLIB::ACOS_F80,
1780 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128),
1786 ExpandFloatRes_Unary(
N,
1788 RTLIB::ASIN_F64, RTLIB::ASIN_F80,
1789 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128),
1795 ExpandFloatRes_Unary(
N,
1797 RTLIB::ATAN_F64, RTLIB::ATAN_F80,
1798 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128),
1804 ExpandFloatRes_Binary(
N,
1806 RTLIB::ATAN2_F64, RTLIB::ATAN2_F80,
1807 RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128),
1813 ExpandFloatRes_Unary(
N,
GetFPLibCall(
N->getValueType(0), RTLIB::CBRT_F32,
1814 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1816 RTLIB::CBRT_PPCF128),
Lo,
Hi);
1819void DAGTypeLegalizer::ExpandFloatRes_FCEIL(
SDNode *
N,
1822 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1823 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1824 RTLIB::CEIL_PPCF128),
Lo,
Hi);
1827void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(
SDNode *
N,
1830 RTLIB::COPYSIGN_F32,
1831 RTLIB::COPYSIGN_F64,
1832 RTLIB::COPYSIGN_F80,
1833 RTLIB::COPYSIGN_F128,
1834 RTLIB::COPYSIGN_PPCF128),
Lo,
Hi);
1837void DAGTypeLegalizer::ExpandFloatRes_FCOS(
SDNode *
N,
1840 RTLIB::COS_F32, RTLIB::COS_F64,
1841 RTLIB::COS_F80, RTLIB::COS_F128,
1842 RTLIB::COS_PPCF128),
Lo,
Hi);
1847 ExpandFloatRes_Unary(
N,
1849 RTLIB::COSH_F64, RTLIB::COSH_F80,
1850 RTLIB::COSH_F128, RTLIB::COSH_PPCF128),
1861 RTLIB::DIV_PPCF128),
Lo,
Hi);
1864void DAGTypeLegalizer::ExpandFloatRes_FEXP(
SDNode *
N,
1867 RTLIB::EXP_F32, RTLIB::EXP_F64,
1868 RTLIB::EXP_F80, RTLIB::EXP_F128,
1869 RTLIB::EXP_PPCF128),
Lo,
Hi);
1872void DAGTypeLegalizer::ExpandFloatRes_FEXP2(
SDNode *
N,
1875 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1876 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1877 RTLIB::EXP2_PPCF128),
Lo,
Hi);
1882 ExpandFloatRes_Unary(
N,
1884 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1885 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1889void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(
SDNode *
N,
1892 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1893 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1894 RTLIB::FLOOR_PPCF128),
Lo,
Hi);
1897void DAGTypeLegalizer::ExpandFloatRes_FLOG(
SDNode *
N,
1900 RTLIB::LOG_F32, RTLIB::LOG_F64,
1901 RTLIB::LOG_F80, RTLIB::LOG_F128,
1902 RTLIB::LOG_PPCF128),
Lo,
Hi);
1905void DAGTypeLegalizer::ExpandFloatRes_FLOG2(
SDNode *
N,
1908 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1909 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1910 RTLIB::LOG2_PPCF128),
Lo,
Hi);
1913void DAGTypeLegalizer::ExpandFloatRes_FLOG10(
SDNode *
N,
1916 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1917 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1918 RTLIB::LOG10_PPCF128),
Lo,
Hi);
1923 bool IsStrict =
N->isStrictFPOpcode();
1924 unsigned Offset = IsStrict ? 1 : 0;
1934 RTLIB::FMA_PPCF128),
1935 N->getValueType(0), Ops, CallOptions,
1938 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1939 GetPairElements(Tmp.first,
Lo,
Hi);
1949 RTLIB::MUL_PPCF128),
Lo,
Hi);
1952void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(
SDNode *
N,
1955 RTLIB::NEARBYINT_F32,
1956 RTLIB::NEARBYINT_F64,
1957 RTLIB::NEARBYINT_F80,
1958 RTLIB::NEARBYINT_F128,
1959 RTLIB::NEARBYINT_PPCF128),
Lo,
Hi);
1965 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1970void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(
SDNode *
N,
SDValue &
Lo,
1974 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1981 bool IsStrict =
N->isStrictFPOpcode();
1986 if (NVT ==
N->getOperand(1).getValueType()) {
1987 Hi =
N->getOperand(1);
1988 Chain =
N->getOperand(0);
1992 {
N->getOperand(0),
N->getOperand(1) });
1993 Chain =
Hi.getValue(1);
2002 ReplaceValueWith(
SDValue(
N, 1), Chain);
2005void DAGTypeLegalizer::ExpandFloatRes_FPOW(
SDNode *
N,
2008 RTLIB::POW_F32, RTLIB::POW_F64,
2009 RTLIB::POW_F80, RTLIB::POW_F128,
2010 RTLIB::POW_PPCF128),
Lo,
Hi);
2013void DAGTypeLegalizer::ExpandFloatRes_FPOWI(
SDNode *
N,
2023void DAGTypeLegalizer::ExpandFloatRes_FREEZE(
SDNode *
N,
2025 assert(
N->getValueType(0) == MVT::ppcf128 &&
2026 "Logic only correct for ppcf128!");
2029 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
2034void DAGTypeLegalizer::ExpandFloatRes_FREM(
SDNode *
N,
2037 RTLIB::REM_F32, RTLIB::REM_F64,
2038 RTLIB::REM_F80, RTLIB::REM_F128,
2039 RTLIB::REM_PPCF128),
Lo,
Hi);
2042void DAGTypeLegalizer::ExpandFloatRes_FRINT(
SDNode *
N,
2045 RTLIB::RINT_F32, RTLIB::RINT_F64,
2046 RTLIB::RINT_F80, RTLIB::RINT_F128,
2047 RTLIB::RINT_PPCF128),
Lo,
Hi);
2050void DAGTypeLegalizer::ExpandFloatRes_FROUND(
SDNode *
N,
2057 RTLIB::ROUND_PPCF128),
Lo,
Hi);
2060void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(
SDNode *
N,
2063 RTLIB::ROUNDEVEN_F32,
2064 RTLIB::ROUNDEVEN_F64,
2065 RTLIB::ROUNDEVEN_F80,
2066 RTLIB::ROUNDEVEN_F128,
2067 RTLIB::ROUNDEVEN_PPCF128),
Lo,
Hi);
2070void DAGTypeLegalizer::ExpandFloatRes_FSIN(
SDNode *
N,
2073 RTLIB::SIN_F32, RTLIB::SIN_F64,
2074 RTLIB::SIN_F80, RTLIB::SIN_F128,
2075 RTLIB::SIN_PPCF128),
Lo,
Hi);
2080 ExpandFloatRes_Unary(
N,
2082 RTLIB::SINH_F64, RTLIB::SINH_F80,
2083 RTLIB::SINH_F128, RTLIB::SINH_PPCF128),
2087void DAGTypeLegalizer::ExpandFloatRes_FSQRT(
SDNode *
N,
2090 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
2091 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
2092 RTLIB::SQRT_PPCF128),
Lo,
Hi);
2102 RTLIB::SUB_PPCF128),
Lo,
Hi);
2107 ExpandFloatRes_Unary(
N,
2109 RTLIB::TAN_F64, RTLIB::TAN_F80,
2110 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
2116 ExpandFloatRes_Unary(
N,
2118 RTLIB::TANH_F64, RTLIB::TANH_F80,
2119 RTLIB::TANH_F128, RTLIB::TANH_PPCF128),
2123void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(
SDNode *
N,
2126 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
2127 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
2128 RTLIB::TRUNC_PPCF128),
Lo,
Hi);
2134 ExpandRes_NormalLoad(
N,
Lo,
Hi);
2146 assert(
LD->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
2149 LD->getMemoryVT(),
LD->getMemOperand());
2152 Chain =
Hi.getValue(1);
2159 ReplaceValueWith(
SDValue(LD, 1), Chain);
2164 assert(
N->getValueType(0) == MVT::ppcf128 &&
"Unsupported XINT_TO_FP!");
2165 EVT VT =
N->getValueType(0);
2167 bool Strict =
N->isStrictFPOpcode();
2168 SDValue Src =
N->getOperand(Strict ? 1 : 0);
2169 EVT SrcVT = Src.getValueType();
2177 Flags.setNoFPExcept(
N->getFlags().hasNoFPExcept());
2182 if (SrcVT.
bitsLE(MVT::i32)) {
2187 {Chain, Src}, Flags);
2188 Chain =
Hi.getValue(1);
2190 Hi = DAG.
getNode(
N->getOpcode(), dl, NVT, Src);
2192 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2193 if (SrcVT.
bitsLE(MVT::i64)) {
2196 LC = RTLIB::SINTTOFP_I64_PPCF128;
2197 }
else if (SrcVT.
bitsLE(MVT::i128)) {
2199 LC = RTLIB::SINTTOFP_I128_PPCF128;
2201 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
2205 std::pair<SDValue, SDValue> Tmp =
2206 TLI.
makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
2209 GetPairElements(Tmp.first,
Lo,
Hi);
2215 ReplaceValueWith(
SDValue(
N, 1), Chain);
2225 SrcVT = Src.getValueType();
2228 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2229 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2230 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2252 {Chain, Hi, NewLo}, Flags);
2253 Chain =
Lo.getValue(1);
2254 ReplaceValueWith(
SDValue(
N, 1), Chain);
2259 GetPairElements(
Lo,
Lo,
Hi);
2271bool DAGTypeLegalizer::ExpandFloatOperand(
SDNode *
N,
unsigned OpNo) {
2276 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
2279 switch (
N->getOpcode()) {
2282 dbgs() <<
"ExpandFloatOperand Op #" << OpNo <<
": ";
2283 N->dump(&DAG);
dbgs() <<
"\n";
2291 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(
N);
break;
2299 case ISD::LROUND: Res = ExpandFloatOp_LROUND(
N);
break;
2301 case ISD::LRINT: Res = ExpandFloatOp_LRINT(
N);
break;
2302 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(
N);
break;
2306 case ISD::SETCC: Res = ExpandFloatOp_SETCC(
N);
break;
2307 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(
N),
2312 if (!Res.
getNode())
return false;
2320 "Invalid operand expansion");
2322 ReplaceValueWith(
SDValue(
N, 0), Res);
2328void DAGTypeLegalizer::FloatExpandSetCCOperands(
SDValue &NewLHS,
2333 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2334 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2335 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2344 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2349 RHSLo, CCCode, OutputChain, IsSignaling);
2357 RHSHi, CCCode, OutputChain, IsSignaling);
2362 Chain = OutputChain;
2366 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
2367 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
2369 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain);
2381 N->getOperand(4)), 0);
2385 assert(
N->getOperand(1).getValueType() == MVT::ppcf128 &&
2386 "Logic only correct for ppcf128!");
2388 GetExpandedFloat(
N->getOperand(1),
Lo,
Hi);
2392 N->getValueType(0),
N->getOperand(0),
Hi);
2396 bool IsStrict =
N->isStrictFPOpcode();
2397 assert(
N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2398 "Logic only correct for ppcf128!");
2400 GetExpandedFloat(
N->getOperand(IsStrict ? 1 : 0),
Lo,
Hi);
2405 N->getValueType(0),
Hi,
N->getOperand(1));
2409 if (
Hi.getValueType() ==
N->getValueType(0)) {
2411 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
2417 {
N->getValueType(0), MVT::Other},
2418 {
N->getOperand(0),
Hi,
N->getOperand(2)});
2425 EVT RVT =
N->getValueType(0);
2428 bool IsStrict =
N->isStrictFPOpcode();
2431 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
2437 "Unsupported FP_TO_XINT!");
2439 std::pair<SDValue, SDValue> Tmp =
2444 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2445 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2450 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
2451 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
2453 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain);
2464 N->getOperand(2),
N->getOperand(3),
2469 bool IsStrict =
N->isStrictFPOpcode();
2470 SDValue NewLHS =
N->getOperand(IsStrict ? 1 : 0);
2471 SDValue NewRHS =
N->getOperand(IsStrict ? 2 : 1);
2474 cast<CondCodeSDNode>(
N->getOperand(IsStrict ? 3 : 2))->get();
2475 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain,
2481 "Unexpected setcc expansion!");
2483 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
2484 ReplaceValueWith(
SDValue(
N, 1), Chain);
2490SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2492 return ExpandOp_NormalStore(
N, OpNo);
2495 assert(OpNo == 1 &&
"Can only expand the stored value so far");
2502 ST->getValue().getValueType());
2504 assert(
ST->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
2508 GetExpandedOp(
ST->getValue(),
Lo,
Hi);
2511 ST->getMemoryVT(),
ST->getMemOperand());
2515 EVT RVT =
N->getValueType(0);
2516 EVT RetVT =
N->getOperand(0).getValueType();
2523 RTLIB::LROUND_PPCF128),
2524 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2528 EVT RVT =
N->getValueType(0);
2529 EVT RetVT =
N->getOperand(0).getValueType();
2535 RTLIB::LLROUND_F128,
2536 RTLIB::LLROUND_PPCF128),
2537 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2541 EVT RVT =
N->getValueType(0);
2542 EVT RetVT =
N->getOperand(0).getValueType();
2549 RTLIB::LRINT_PPCF128),
2550 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2554 EVT RVT =
N->getValueType(0);
2555 EVT RetVT =
N->getOperand(0).getValueType();
2562 RTLIB::LLRINT_PPCF128),
2563 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2572 if (OpVT == MVT::f16)
2574 if (RetVT == MVT::f16)
2576 if (OpVT == MVT::bf16)
2578 if (RetVT == MVT::bf16)
2584 if (OpVT == MVT::f16)
2586 if (RetVT == MVT::f16)
2588 if (OpVT == MVT::bf16)
2590 if (RetVT == MVT::bf16)
2595bool DAGTypeLegalizer::PromoteFloatOperand(
SDNode *
N,
unsigned OpNo) {
2596 LLVM_DEBUG(
dbgs() <<
"Promote float operand " << OpNo <<
": ";
N->dump(&DAG));
2599 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
2610 switch (
N->getOpcode()) {
2613 dbgs() <<
"PromoteFloatOperand Op #" << OpNo <<
": ";
2614 N->dump(&DAG);
dbgs() <<
"\n";
2620 R = PromoteFloatOp_FAKE_USE(
N, OpNo);
2628 case ISD::LLRINT:
R = PromoteFloatOp_UnaryOp(
N, OpNo);
break;
2632 R = PromoteFloatOp_FP_TO_XINT_SAT(
N, OpNo);
break;
2635 R = PromoteFloatOp_STRICT_FP_EXTEND(
N, OpNo);
2638 case ISD::SETCC:
R = PromoteFloatOp_SETCC(
N, OpNo);
break;
2639 case ISD::STORE:
R = PromoteFloatOp_STORE(
N, OpNo);
break;
2645 ReplaceValueWith(
SDValue(
N, 0), R);
2649SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(
SDNode *
N,
unsigned OpNo) {
2651 EVT OpVT =
Op->getValueType(0);
2653 SDValue Promoted = GetPromotedFloat(
N->getOperand(0));
2662 return DAG.
getBitcast(
N->getValueType(0), Convert);
2665SDValue DAGTypeLegalizer::PromoteFloatOp_FAKE_USE(
SDNode *
N,
unsigned OpNo) {
2666 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
2667 SDValue Op = GetPromotedFloat(
N->getOperand(OpNo));
2668 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::Other,
N->getOperand(0),
2674SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(
SDNode *
N,
unsigned OpNo) {
2675 assert (OpNo == 1 &&
"Only Operand 1 must need promotion here");
2676 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2679 N->getOperand(0), Op1);
2683SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(
SDNode *
N,
unsigned OpNo) {
2684 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2689SDValue DAGTypeLegalizer::PromoteFloatOp_AssertNoFPClass(
SDNode *
N,
2691 return GetPromotedFloat(
N->getOperand(0));
2694SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(
SDNode *
N,
2696 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2701SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(
SDNode *
N,
unsigned OpNo) {
2702 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2703 EVT VT =
N->getValueType(0);
2706 if (VT ==
Op->getValueType(0))
2713SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(
SDNode *
N,
2715 assert(OpNo == 1 &&
"Promoting unpromotable operand");
2717 SDValue Op = GetPromotedFloat(
N->getOperand(1));
2718 EVT VT =
N->getValueType(0);
2721 if (VT ==
Op->getValueType(0)) {
2722 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
2728 N->getOperand(0),
Op);
2736SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2741 LHS, RHS,
N->getOperand(2),
N->getOperand(3),
2747SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2748 EVT VT =
N->getValueType(0);
2749 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2750 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2751 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
2759SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2764 SDValue Promoted = GetPromotedFloat(Val);
2765 EVT VT =
ST->getOperand(1).getValueType();
2772 return DAG.
getStore(
ST->getChain(),
DL, NewVal,
ST->getBasePtr(),
2773 ST->getMemOperand());
2782 SDValue Promoted = GetPromotedFloat(Val);
2783 EVT VT =
ST->getOperand(1).getValueType();
2790 ST->getBasePtr(),
ST->getMemOperand());
2797void DAGTypeLegalizer::PromoteFloatResult(
SDNode *
N,
unsigned ResNo) {
2798 LLVM_DEBUG(
dbgs() <<
"Promote float result " << ResNo <<
": ";
N->dump(&DAG));
2802 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
2807 switch (
N->getOpcode()) {
2814 dbgs() <<
"PromoteFloatResult #" << ResNo <<
": ";
2815 N->dump(&DAG);
dbgs() <<
"\n";
2820 R = PromoteFloatRes_BITCAST(
N);
2823 R = PromoteFloatRes_FREEZE(
N);
2827 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(
N);
break;
2859 R = PromoteFloatRes_AssertNoFPClass(
N);
2877 case ISD::FSUB:
R = PromoteFloatRes_BinOp(
N);
break;
2880 case ISD::FMAD:
R = PromoteFloatRes_FMAD(
N);
break;
2889 R = PromoteFloatRes_UnaryWithTwoFPResults(
N);
2893 R = PromoteFloatRes_STRICT_FP_ROUND(
N);
2895 case ISD::LOAD:
R = PromoteFloatRes_LOAD(
N);
break;
2897 R = PromoteFloatRes_ATOMIC_LOAD(
N);
2913 R = PromoteFloatRes_VECREDUCE(
N);
2917 R = PromoteFloatRes_VECREDUCE_SEQ(
N);
2922 SetPromotedFloat(
SDValue(
N, ResNo), R);
2931 EVT VT =
N->getValueType(0);
2936 N->getOperand(0).getValueType().getSizeInBits());
2942 EVT VT =
N->getValueType(0);
2947 N->getOperand(0).getValueType().getSizeInBits());
2955 EVT VT =
N->getValueType(0);
2974SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
2979 if (isa<ConstantSDNode>(
N->getOperand(1))) {
2987 switch (getTypeAction(VecVT)) {
2990 SDValue Res = GetScalarizedVector(
N->getOperand(0));
2991 ReplaceValueWith(
SDValue(
N, 0), Res);
2995 Vec = GetWidenedVector(Vec);
2997 ReplaceValueWith(
SDValue(
N, 0), Res);
3002 GetSplitVector(Vec,
Lo,
Hi);
3004 uint64_t LoElts =
Lo.getValueType().getVectorNumElements();
3006 if (IdxVal < LoElts)
3011 Idx.getValueType()));
3012 ReplaceValueWith(
SDValue(
N, 0), Res);
3020 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
3025 NewOp,
N->getOperand(1));
3028 EVT VT =
N->getValueType(0);
3037 EVT VT =
N->getValueType(0);
3039 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3050 EVT VT =
N->getValueType(0);
3052 SDValue Op = GetPromotedFloat(
N->getOperand(0));
3059SDValue DAGTypeLegalizer::PromoteFloatRes_AssertNoFPClass(
SDNode *
N) {
3060 return GetPromotedFloat(
N->getOperand(0));
3067 EVT VT =
N->getValueType(0);
3069 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3070 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
3071 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), NVT, Op0, Op1,
N->getFlags());
3075 EVT VT =
N->getValueType(0);
3077 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3078 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
3079 SDValue Op2 = GetPromotedFloat(
N->getOperand(2));
3081 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), NVT, Op0, Op1, Op2);
3086 EVT VT =
N->getValueType(0);
3088 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3095 EVT VT =
N->getValueType(0);
3097 SDValue Op = GetPromotedFloat(
N->getOperand(0));
3105SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryWithTwoFPResults(
SDNode *
N) {
3106 EVT VT =
N->getValueType(0);
3108 SDValue Op = GetPromotedFloat(
N->getOperand(0));
3111 for (
unsigned ResNum = 0, NumValues =
N->getNumValues(); ResNum < NumValues;
3125 EVT VT =
N->getValueType(0);
3126 EVT OpVT =
Op->getValueType(0);
3138SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(
SDNode *
N) {
3143 EVT VT =
N->getValueType(0);
3144 EVT OpVT =
Op->getValueType(0);
3161 EVT VT =
N->getValueType(0);
3166 L->getAddressingMode(),
L->getExtensionType(), IVT,
SDLoc(
N),
3167 L->getChain(),
L->getBasePtr(),
L->getOffset(),
L->getPointerInfo(), IVT,
3168 L->getBaseAlign(),
L->getMemOperand()->getFlags(),
L->getAAInfo());
3178SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(
SDNode *
N) {
3203 N->getOperand(0), TrueVal, FalseVal);
3213 TrueVal.getNode()->getValueType(0),
N->getOperand(0),
3214 N->getOperand(1), TrueVal, FalseVal,
N->getOperand(4));
3221 EVT VT =
N->getValueType(0);
3233 N->getValueType(0)));
3245SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
3251 EVT VT =
N->getValueType(0);
3262 { AM->getChain(), AM->getBasePtr(), CastVal },
3285void DAGTypeLegalizer::SoftPromoteHalfResult(
SDNode *
N,
unsigned ResNo) {
3286 LLVM_DEBUG(
dbgs() <<
"Soft promote half result " << ResNo <<
": ";
3291 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
3296 switch (
N->getOpcode()) {
3299 dbgs() <<
"SoftPromoteHalfResult #" << ResNo <<
": ";
3300 N->dump(&DAG);
dbgs() <<
"\n";
3306 R = SoftPromoteHalfRes_ARITH_FENCE(
N);
break;
3310 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
N);
break;
3345 R = SoftPromoteHalfRes_AssertNoFPClass(
N);
3361 case ISD::FSUB:
R = SoftPromoteHalfRes_BinOp(
N);
break;
3364 case ISD::FMAD:
R = SoftPromoteHalfRes_FMAD(
N);
break;
3374 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(
N);
3377 case ISD::LOAD:
R = SoftPromoteHalfRes_LOAD(
N);
break;
3379 R = SoftPromoteHalfRes_ATOMIC_LOAD(
N);
3388 case ISD::UNDEF:
R = SoftPromoteHalfRes_UNDEF(
N);
break;
3396 R = SoftPromoteHalfRes_VECREDUCE(
N);
3400 R = SoftPromoteHalfRes_VECREDUCE_SEQ(
N);
3405 SetSoftPromotedHalf(
SDValue(
N, ResNo), R);
3408SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(
SDNode *
N) {
3410 BitConvertToInteger(
N->getOperand(0)));
3414 return BitConvertToInteger(
N->getOperand(0));
3417SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(
SDNode *
N) {
3425SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
3426 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
3432SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(
SDNode *
N) {
3433 SDValue LHS = GetSoftPromotedHalf(
N->getOperand(0));
3434 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
3437 EVT LVT =
LHS.getValueType();
3438 EVT RVT =
RHS.getValueType();
3459 }
else if (SizeDiff < 0) {
3481 EVT OVT =
N->getValueType(0);
3483 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3484 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
3485 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
3490 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3491 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3492 Op2 = DAG.
getNode(PromotionOpcode, dl, NVT, Op2);
3501 EVT OVT =
N->getValueType(0);
3503 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3517 EVT OVT =
N->getValueType(0);
3519 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
3534SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(
SDNode *
N) {
3535 EVT OVT =
N->getValueType(0);
3537 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
3546 for (
unsigned ResNum = 0, NumValues =
N->getNumValues(); ResNum < NumValues;
3549 SetSoftPromotedHalf(
SDValue(
N, ResNum), Trunc);
3555SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(
SDNode *
N) {
3556 EVT RVT =
N->getValueType(0);
3557 bool IsStrict =
N->isStrictFPOpcode();
3558 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3559 EVT SVT =
Op.getValueType();
3565 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND libcall");
3568 Op = GetSoftenedFloat(
Op);
3571 std::pair<SDValue, SDValue> Tmp =
3574 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3580 {MVT::i16, MVT::Other}, {
N->getOperand(0),
Op});
3595 DAG.
getLoad(
L->getAddressingMode(),
L->getExtensionType(), MVT::i16,
3596 SDLoc(
N),
L->getChain(),
L->getBasePtr(),
L->getOffset(),
3597 L->getPointerInfo(), MVT::i16,
L->getBaseAlign(),
3598 L->getMemOperand()->getFlags(),
L->getAAInfo());
3605SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(
SDNode *
N) {
3620 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
3621 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
3626SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(
SDNode *
N) {
3627 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
3628 SDValue Op3 = GetSoftPromotedHalf(
N->getOperand(3));
3630 N->getOperand(0),
N->getOperand(1), Op2, Op3,
3634SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(
SDNode *
N) {
3635 EVT OVT =
N->getValueType(0);
3639 if (
N->isStrictFPOpcode()) {
3641 {N->getOperand(0), N->getOperand(1)});
3643 {MVT::i16, MVT::Other}, {
Op.getValue(1),
Op});
3644 ReplaceValueWith(
SDValue(
N, 1),
Op.getValue(1));
3659 EVT OVT =
N->getValueType(0);
3661 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
3673SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(
SDNode *
N) {
3674 return GetSoftPromotedHalf(
N->getOperand(0));
3678 EVT OVT =
N->getValueType(0);
3680 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3681 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
3686 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3687 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3695SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(
SDNode *
N) {
3701SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(
SDNode *
N) {
3711bool DAGTypeLegalizer::SoftPromoteHalfOperand(
SDNode *
N,
unsigned OpNo) {
3712 LLVM_DEBUG(
dbgs() <<
"Soft promote half operand " << OpNo <<
": ";
3716 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
3726 switch (
N->getOpcode()) {
3729 dbgs() <<
"SoftPromoteHalfOperand Op #" << OpNo <<
": ";
3730 N->dump(&DAG);
dbgs() <<
"\n";
3735 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(
N);
break;
3737 Res = SoftPromoteHalfOp_FAKE_USE(
N, OpNo);
3739 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(
N, OpNo);
break;
3746 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(
N);
break;
3749 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(
N, OpNo);
break;
3750 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(
N);
break;
3751 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(
N, OpNo);
break;
3753 Res = SoftPromoteHalfOp_ATOMIC_STORE(
N, OpNo);
3756 Res = SoftPromoteHalfOp_STACKMAP(
N, OpNo);
3759 Res = SoftPromoteHalfOp_PATCHPOINT(
N, OpNo);
3769 "Invalid operand expansion");
3771 ReplaceValueWith(
SDValue(
N, 0), Res);
3776 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3781SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(
SDNode *
N,
unsigned OpNo) {
3782 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
3783 SDValue Op = GetSoftPromotedHalf(
N->getOperand(OpNo));
3784 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::Other,
N->getOperand(0),
3790 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
3797 Op1 = GetSoftPromotedHalf(Op1);
3800 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0),
N->getOperand(0),
3804SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(
SDNode *
N) {
3805 EVT RVT =
N->getValueType(0);
3806 bool IsStrict =
N->isStrictFPOpcode();
3807 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3808 EVT SVT =
Op.getValueType();
3809 Op = GetSoftPromotedHalf(
N->getOperand(IsStrict ? 1 : 0));
3813 {RVT, MVT::Other}, {
N->getOperand(0),
Op});
3815 ReplaceValueWith(
SDValue(
N, 0), Res);
3822SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(
SDNode *
N) {
3823 EVT RVT =
N->getValueType(0);
3824 bool IsStrict =
N->isStrictFPOpcode();
3825 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3826 EVT SVT =
Op.getValueType();
3830 Op = GetSoftPromotedHalf(
Op);
3834 {
N->getOperand(0),
Op});
3835 Op = DAG.
getNode(
N->getOpcode(), dl, {RVT, MVT::Other},
3836 {Op.getValue(1), Op});
3837 ReplaceValueWith(
SDValue(
N, 1),
Op.getValue(1));
3843 return DAG.
getNode(
N->getOpcode(), dl, RVT, Res);
3846SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(
SDNode *
N) {
3847 EVT RVT =
N->getValueType(0);
3849 EVT SVT =
Op.getValueType();
3854 Op = GetSoftPromotedHalf(
Op);
3858 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0), Res,
3864 assert(OpNo == 0 &&
"Can only soften the comparison values");
3872 Op0 = GetSoftPromotedHalf(Op0);
3873 Op1 = GetSoftPromotedHalf(Op1);
3877 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3878 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3881 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
3887 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
3893 Op0 = GetSoftPromotedHalf(Op0);
3894 Op1 = GetSoftPromotedHalf(Op1);
3898 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3899 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3904SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(
SDNode *
N,
unsigned OpNo) {
3905 assert(OpNo == 1 &&
"Can only soften the stored value!");
3910 assert(!
ST->isTruncatingStore() &&
"Unexpected truncating store.");
3911 SDValue Promoted = GetSoftPromotedHalf(Val);
3912 return DAG.
getStore(
ST->getChain(), dl, Promoted,
ST->getBasePtr(),
3913 ST->getMemOperand());
3916SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(
SDNode *
N,
3918 assert(OpNo == 1 &&
"Can only soften the stored value!");
3923 SDValue Promoted = GetSoftPromotedHalf(Val);
3925 ST->getChain(), Promoted,
ST->getBasePtr(),
3926 ST->getMemOperand());
3929SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
3933 NewOps[OpNo] = GetSoftPromotedHalf(
Op);
3937 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
3948 NewOps[OpNo] = GetSoftPromotedHalf(
Op);
3952 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
APInt bitcastToAPInt() const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool expandMultipleResultFPLibCall(RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={})
Expands a node with multiple results to an FP or vector libcall.
LLVMContext * getContext() const
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
The instances of the Type class are immutable: once they are created, they are never changed.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ SHL
Shift and rotation operations.
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum maximum on two values, following IEEE-754 definition...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
LLVM_ABI Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOSPI(EVT RetVT)
getSINCOSPI - Return the SINCOSPI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getMODF(EVT RetVT)
getMODF - Return the MODF_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getCOS(EVT RetVT)
Return the COS_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSIN(EVT RetVT)
Return the SIN_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getPOW(EVT RetVT)
getPOW - Return the POW_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOS(EVT RetVT)
getSINCOS - Return the SINCOS_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
DWARFExpression::Operation Op
static LLVM_ABI const fltSemantics & PPCDoubleDouble() LLVM_READNONE
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)