27#define DEBUG_TYPE "legalize-types"
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
51void DAGTypeLegalizer::SoftenFloatResult(
SDNode *
N,
unsigned ResNo) {
52 LLVM_DEBUG(
dbgs() <<
"Soften float result " << ResNo <<
": ";
N->dump(&DAG));
55 switch (
N->getOpcode()) {
59 dbgs() <<
"SoftenFloatResult #" << ResNo <<
": ";
60 N->dump(&DAG);
dbgs() <<
"\n";
65 case ISD::ARITH_FENCE:
R = SoftenFloatRes_ARITH_FENCE(
N);
break;
67 case ISD::BITCAST:
R = SoftenFloatRes_BITCAST(
N);
break;
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(
N, ResNo);
break;
72 case ISD::FABS:
R = SoftenFloatRes_FABS(
N);
break;
74 case ISD::FMINNUM:
R = SoftenFloatRes_FMINNUM(
N);
break;
76 case ISD::FMAXNUM:
R = SoftenFloatRes_FMAXNUM(
N);
break;
77 case ISD::FMINIMUMNUM:
R = SoftenFloatRes_FMINIMUMNUM(
N);
break;
78 case ISD::FMAXIMUMNUM:
R = SoftenFloatRes_FMAXIMUMNUM(
N);
break;
79 case ISD::FMINIMUM:
R = SoftenFloatRes_FMINIMUM(
N);
break;
80 case ISD::FMAXIMUM:
R = SoftenFloatRes_FMAXIMUM(
N);
break;
84 case ISD::FACOS:
R = SoftenFloatRes_FACOS(
N);
break;
86 case ISD::FASIN:
R = SoftenFloatRes_FASIN(
N);
break;
88 case ISD::FATAN:
R = SoftenFloatRes_FATAN(
N);
break;
90 case ISD::FATAN2:
R = SoftenFloatRes_FATAN2(
N);
break;
91 case ISD::FCBRT:
R = SoftenFloatRes_FCBRT(
N);
break;
93 case ISD::FCEIL:
R = SoftenFloatRes_FCEIL(
N);
break;
96 case ISD::FCOS:
R = SoftenFloatRes_FCOS(
N);
break;
98 case ISD::FCOSH:
R = SoftenFloatRes_FCOSH(
N);
break;
102 case ISD::FEXP:
R = SoftenFloatRes_FEXP(
N);
break;
104 case ISD::FEXP2:
R = SoftenFloatRes_FEXP2(
N);
break;
105 case ISD::FEXP10:
R = SoftenFloatRes_FEXP10(
N);
break;
107 case ISD::FFLOOR:
R = SoftenFloatRes_FFLOOR(
N);
break;
109 case ISD::FLOG:
R = SoftenFloatRes_FLOG(
N);
break;
111 case ISD::FLOG2:
R = SoftenFloatRes_FLOG2(
N);
break;
113 case ISD::FLOG10:
R = SoftenFloatRes_FLOG10(
N);
break;
115 case ISD::FMA:
R = SoftenFloatRes_FMA(
N);
break;
119 case ISD::FNEARBYINT:
R = SoftenFloatRes_FNEARBYINT(
N);
break;
120 case ISD::FNEG:
R = SoftenFloatRes_FNEG(
N);
break;
122 case ISD::FP_EXTEND:
R = SoftenFloatRes_FP_EXTEND(
N);
break;
125 case ISD::FP16_TO_FP:
R = SoftenFloatRes_FP16_TO_FP(
N);
break;
126 case ISD::BF16_TO_FP:
R = SoftenFloatRes_BF16_TO_FP(
N);
break;
128 case ISD::FPOW:
R = SoftenFloatRes_FPOW(
N);
break;
133 case ISD::FFREXP:
R = SoftenFloatRes_FFREXP(
N);
break;
134 case ISD::FSINCOS:
R = SoftenFloatRes_FSINCOS(
N);
break;
135 case ISD::FMODF:
R = SoftenFloatRes_FMODF(
N);
break;
139 case ISD::FRINT:
R = SoftenFloatRes_FRINT(
N);
break;
141 case ISD::FROUND:
R = SoftenFloatRes_FROUND(
N);
break;
143 case ISD::FROUNDEVEN:
R = SoftenFloatRes_FROUNDEVEN(
N);
break;
145 case ISD::FSIN:
R = SoftenFloatRes_FSIN(
N);
break;
147 case ISD::FSINH:
R = SoftenFloatRes_FSINH(
N);
break;
149 case ISD::FSQRT:
R = SoftenFloatRes_FSQRT(
N);
break;
153 case ISD::FTAN:
R = SoftenFloatRes_FTAN(
N);
break;
155 case ISD::FTANH:
R = SoftenFloatRes_FTANH(
N);
break;
157 case ISD::FTRUNC:
R = SoftenFloatRes_FTRUNC(
N);
break;
158 case ISD::LOAD:
R = SoftenFloatRes_LOAD(
N);
break;
159 case ISD::ATOMIC_LOAD:
R = SoftenFloatRes_ATOMIC_LOAD(
N);
break;
160 case ISD::ATOMIC_SWAP:
R = BitcastToInt_ATOMIC_SWAP(
N);
break;
170 case ISD::VAARG:
R = SoftenFloatRes_VAARG(
N);
break;
172 case ISD::VECREDUCE_FADD:
173 case ISD::VECREDUCE_FMUL:
174 case ISD::VECREDUCE_FMIN:
175 case ISD::VECREDUCE_FMAX:
176 case ISD::VECREDUCE_FMAXIMUM:
177 case ISD::VECREDUCE_FMINIMUM:
R = SoftenFloatRes_VECREDUCE(
N);
break;
178 case ISD::VECREDUCE_SEQ_FADD:
179 case ISD::VECREDUCE_SEQ_FMUL:
R = SoftenFloatRes_VECREDUCE_SEQ(
N);
break;
186 SetSoftenedFloat(
SDValue(
N, ResNo), R);
190SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(
SDNode *
N, RTLIB::Libcall LC) {
192 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
193 unsigned Offset = IsStrict ? 1 : 0;
195 "Unexpected number of operands!");
198 TargetLowering::MakeLibCallOptions CallOptions;
199 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT,
Op,
202 CallOptions, SDLoc(
N),
205 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
209SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(
SDNode *
N, RTLIB::Libcall LC) {
210 bool IsStrict =
N->isStrictFPOpcode();
211 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
212 unsigned Offset = IsStrict ? 1 : 0;
214 "Unexpected number of operands!");
216 GetSoftenedFloat(
N->getOperand(1 +
Offset)) };
218 TargetLowering::MakeLibCallOptions CallOptions;
219 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
220 N->getOperand(1 +
Offset).getValueType() };
222 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT,
Ops,
223 CallOptions, SDLoc(
N),
226 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
231 return BitConvertToInteger(
N->getOperand(0));
235 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
237 GetSoftenedFloat(
N->getOperand(0)));
241 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
242 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(
N), Ty,
243 GetSoftenedFloat(
N->getOperand(0)));
249 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
250 return BitConvertToInteger(
Op);
256 TLI.getTypeToTransformTo(*DAG.getContext(),
258 BitConvertToInteger(
N->getOperand(0)),
259 BitConvertToInteger(
N->getOperand(1)));
271 if (DAG.getDataLayout().isBigEndian() &&
275 APInt Val(128, words);
276 return DAG.getConstant(Val, SDLoc(CN),
277 TLI.getTypeToTransformTo(*DAG.getContext(),
281 TLI.getTypeToTransformTo(*DAG.getContext(),
286SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(
SDNode *
N) {
288 assert(Src.getValueType() == MVT::ppcf128 &&
289 "In floats only ppcf128 can be extracted by element!");
291 N->getValueType(0).changeTypeToInteger(),
292 DAG.getBitcast(MVT::i128, Src),
N->getOperand(1));
295SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N,
unsigned ResNo) {
296 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
299 NewOp,
N->getOperand(1));
303 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
310 SDValue Op = GetSoftenedFloat(
N->getOperand(0));
311 return DAG.getNode(
ISD::AND, SDLoc(
N), NVT,
Op, Mask);
315 if (
SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(
N, DAG))
316 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
322 RTLIB::FMIN_PPCF128));
326 if (
SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(
N, DAG))
327 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
333 RTLIB::FMAX_PPCF128));
337 return SoftenFloatRes_Binary(
339 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
340 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128));
344 return SoftenFloatRes_Binary(
346 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
347 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128));
351 return SoftenFloatRes_Binary(
353 RTLIB::FMINIMUM_F64, RTLIB::FMINIMUM_F80,
354 RTLIB::FMINIMUM_F128, RTLIB::FMINIMUM_PPCF128));
358 return SoftenFloatRes_Binary(
360 RTLIB::FMAXIMUM_F64, RTLIB::FMAXIMUM_F80,
361 RTLIB::FMAXIMUM_F128, RTLIB::FMAXIMUM_PPCF128));
370 RTLIB::ADD_PPCF128));
374 return SoftenFloatRes_Unary(
375 N,
GetFPLibCall(
N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64,
376 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128));
380 return SoftenFloatRes_Unary(
381 N,
GetFPLibCall(
N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64,
382 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128));
386 return SoftenFloatRes_Unary(
387 N,
GetFPLibCall(
N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64,
388 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128));
392 return SoftenFloatRes_Binary(
394 GetFPLibCall(
N->getValueType(0), RTLIB::ATAN2_F32, RTLIB::ATAN2_F64,
395 RTLIB::ATAN2_F80, RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128));
404 RTLIB::CBRT_PPCF128));
413 RTLIB::CEIL_PPCF128));
418 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
421 EVT LVT =
LHS.getValueType();
422 EVT RVT =
RHS.getValueType();
429 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
430 DAG.getConstant(RSize - 1, dl,
431 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
439 DAG.getConstant(SizeDiff, dl,
441 DAG.getDataLayout())));
443 }
else if (SizeDiff < 0) {
447 DAG.getConstant(-SizeDiff, dl,
449 DAG.getDataLayout())));
454 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
455 DAG.getConstant(LSize - 1, dl,
456 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
457 Mask = DAG.getNode(
ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
461 return DAG.getNode(
ISD::OR, dl, LVT,
LHS, SignBit);
470 RTLIB::COS_PPCF128));
474 return SoftenFloatRes_Unary(
475 N,
GetFPLibCall(
N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64,
476 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128));
485 RTLIB::DIV_PPCF128));
494 RTLIB::EXP_PPCF128));
503 RTLIB::EXP2_PPCF128));
507 return SoftenFloatRes_Unary(
509 GetFPLibCall(
N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
510 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
519 RTLIB::FLOOR_PPCF128));
528 RTLIB::LOG_PPCF128));
537 RTLIB::LOG2_PPCF128));
546 RTLIB::LOG10_PPCF128));
550 bool IsStrict =
N->isStrictFPOpcode();
551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
552 unsigned Offset = IsStrict ? 1 : 0;
554 GetSoftenedFloat(
N->getOperand(1 +
Offset)),
555 GetSoftenedFloat(
N->getOperand(2 +
Offset)) };
557 TargetLowering::MakeLibCallOptions CallOptions;
558 EVT OpsVT[3] = {
N->getOperand(0 +
Offset).getValueType(),
559 N->getOperand(1 +
Offset).getValueType(),
560 N->getOperand(2 +
Offset).getValueType() };
562 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
569 NVT,
Ops, CallOptions, SDLoc(
N), Chain);
571 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
581 RTLIB::MUL_PPCF128));
586 RTLIB::NEARBYINT_F32,
587 RTLIB::NEARBYINT_F64,
588 RTLIB::NEARBYINT_F80,
589 RTLIB::NEARBYINT_F128,
590 RTLIB::NEARBYINT_PPCF128));
594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
599 return DAG.getNode(
ISD::XOR, dl, NVT, GetSoftenedFloat(
N->getOperand(0)),
600 DAG.getConstant(SignMask, dl, NVT));
604 bool IsStrict =
N->isStrictFPOpcode();
605 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
611 Op = GetPromotedFloat(
Op);
614 if (
Op.getValueType() ==
N->getValueType(0)) {
616 ReplaceValueWith(
SDValue(
N, 1), Chain);
617 return BitConvertToInteger(
Op);
625 if ((
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16) &&
626 N->getValueType(0) != MVT::f32) {
629 { MVT::f32, MVT::Other }, { Chain,
Op });
630 Chain =
Op.getValue(1);
632 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(
N), MVT::f32,
Op);
636 if (
Op.getValueType() == MVT::bf16) {
638 return SoftenFloatRes_BF16_TO_FP(
N);
642 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
643 TargetLowering::MakeLibCallOptions CallOptions;
644 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
646 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT,
Op,
647 CallOptions, SDLoc(
N),
650 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
657 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
659 TargetLowering::MakeLibCallOptions CallOptions;
660 EVT OpsVT[1] = {
N->getOperand(0).getValueType() };
662 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT,
Op,
663 CallOptions, SDLoc(
N)).first;
664 if (
N->getValueType(0) == MVT::f32)
667 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
669 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
670 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(
N)).first;
676 assert(
N->getValueType(0) == MVT::f32 &&
677 "Can only soften BF16_TO_FP with f32 result");
678 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
682 DAG.getNode(ISD::BITCAST,
DL, MVT::i16,
Op));
684 DAG.getShiftAmountConstant(16, NVT,
DL));
689 bool IsStrict =
N->isStrictFPOpcode();
690 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
694 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND!");
695 TargetLowering::MakeLibCallOptions CallOptions;
696 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
698 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT,
Op,
699 CallOptions, SDLoc(
N),
702 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
711 bool IsStrict =
N->isStrictFPOpcode();
712 unsigned Offset = IsStrict ? 1 : 0;
715 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
719 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
720 if (!TLI.getLibcallName(LC)) {
723 DAG.getContext()->emitError(
"do not know how to soften fpowi to fpow");
725 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
726 return DAG.getPOISON(NVT);
729 if (DAG.getLibInfo().getIntSize() !=
730 N->getOperand(1 +
Offset).getValueType().getSizeInBits()) {
733 DAG.getContext()->emitError(
"powi exponent does not match sizeof(int)");
735 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
736 return DAG.getPOISON(NVT);
742 TargetLowering::MakeLibCallOptions CallOptions;
743 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
744 N->getOperand(1 +
Offset).getValueType() };
746 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT,
Ops,
747 CallOptions, SDLoc(
N),
750 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
755 assert(!
N->isStrictFPOpcode() &&
"strictfp not implemented for frexp");
756 EVT VT0 =
N->getValueType(0);
757 EVT VT1 =
N->getValueType(1);
759 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
766 DAG.getContext()->emitError(
"ffrexp exponent does not match sizeof(int)");
767 SDValue PoisonExp = DAG.getPOISON(VT1);
768 ReplaceValueWith(
SDValue(
N, 1), PoisonExp);
769 return DAG.getMergeValues({DAG.getPOISON(NVT0), PoisonExp},
DL);
772 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
775 TargetLowering::MakeLibCallOptions CallOptions;
776 SDValue Ops[2] = {GetSoftenedFloat(
N->getOperand(0)), StackSlot};
783 .setOpsTypeOverrides(CallOpsTypeOverrides);
785 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0,
Ops, CallOptions,
DL,
791 SDValue LoadExp = DAG.getLoad(VT1,
DL, Chain, StackSlot, PtrInfo);
793 ReplaceValueWith(
SDValue(
N, 1), LoadExp);
797bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
798 SDNode *
N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
799 assert(!
N->isStrictFPOpcode() &&
"strictfp not implemented");
800 EVT VT =
N->getValueType(0);
802 assert(VT ==
N->getValueType(1) &&
803 "expected both return values to have the same type");
805 if (!TLI.getLibcallName(LC))
808 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
815 std::array<SDValue, 2> StackSlots;
818 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ++ResNum) {
819 if (ResNum == CallRetResNo)
821 SDValue StackSlot = DAG.CreateStackTemporary(NVT);
822 Ops.push_back(StackSlot);
824 StackSlots[ResNum] = StackSlot;
828 TargetLowering::MakeLibCallOptions CallOptions;
832 .setOpsTypeOverrides(CallOpsTypeOverrides);
834 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT,
Ops, CallOptions,
DL,
837 auto CreateStackLoad = [&, Chain = Chain](
SDValue StackSlot) {
841 return DAG.getLoad(NVT,
DL, Chain, StackSlot, PtrInfo);
844 for (
auto [ResNum, SlackSlot] :
enumerate(StackSlots)) {
845 if (CallRetResNo == ResNum) {
846 SetSoftenedFloat(
SDValue(
N, ResNum), ReturnVal);
849 SetSoftenedFloat(
SDValue(
N, ResNum), CreateStackLoad(SlackSlot));
856 EVT VT =
N->getValueType(0);
865 if (!TLI.getLibcallName(SinLC) || !TLI.getLibcallName(CosLC)) {
866 DAG.getContext()->emitError(
"do not know how to soften fsincos");
868 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
869 SoftSin = SoftCos = DAG.getPOISON(NVT);
871 SoftSin = SoftenFloatRes_Unary(
N, SinLC);
872 SoftCos = SoftenFloatRes_Unary(
N, CosLC);
875 SetSoftenedFloat(
SDValue(
N, 0), SoftSin);
876 SetSoftenedFloat(
SDValue(
N, 1), SoftCos);
881 EVT VT =
N->getValueType(0);
886 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
887 DAG.getContext()->emitError(
"do not know how to soften fmodf");
900 RTLIB::REM_PPCF128));
909 RTLIB::RINT_PPCF128));
918 RTLIB::ROUND_PPCF128));
923 RTLIB::ROUNDEVEN_F32,
924 RTLIB::ROUNDEVEN_F64,
925 RTLIB::ROUNDEVEN_F80,
926 RTLIB::ROUNDEVEN_F128,
927 RTLIB::ROUNDEVEN_PPCF128));
936 RTLIB::SIN_PPCF128));
940 return SoftenFloatRes_Unary(
941 N,
GetFPLibCall(
N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64,
942 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128));
951 RTLIB::SQRT_PPCF128));
960 RTLIB::SUB_PPCF128));
964 return SoftenFloatRes_Unary(
965 N,
GetFPLibCall(
N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
966 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
970 return SoftenFloatRes_Unary(
971 N,
GetFPLibCall(
N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64,
972 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128));
981 RTLIB::TRUNC_PPCF128));
986 EVT VT =
N->getValueType(0);
987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
991 L->getMemOperand()->getFlags() &
995 NewL = DAG.getLoad(
L->getAddressingMode(),
L->getExtensionType(), NVT, dl,
996 L->getChain(),
L->getBasePtr(),
L->getOffset(),
997 L->getPointerInfo(), NVT,
L->getBaseAlign(), MMOFlags,
1007 dl,
L->getChain(),
L->getBasePtr(),
L->getOffset(),
1008 L->getPointerInfo(),
L->getMemoryVT(),
L->getBaseAlign(),
1009 MMOFlags,
L->getAAInfo());
1013 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
1014 return BitConvertToInteger(ExtendNode);
1019 EVT VT =
N->getValueType(0);
1020 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1025 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
1026 {L->getChain(), L->getBasePtr()},
L->getMemOperand());
1040 return DAG.getSelect(SDLoc(
N),
1041 LHS.getValueType(),
N->getOperand(0),
LHS,
RHS);
1048 LHS.getValueType(),
N->getOperand(0),
1049 N->getOperand(1),
LHS,
RHS,
N->getOperand(4));
1053 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1054 N->getValueType(0)));
1060 EVT VT =
N->getValueType(0);
1061 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1065 NewVAARG = DAG.getVAArg(NVT, dl, Chain,
Ptr,
N->getOperand(2),
1066 N->getConstantOperandVal(3));
1076 bool IsStrict =
N->isStrictFPOpcode();
1079 EVT SVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
1080 EVT RVT =
N->getValueType(0);
1087 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1088 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1089 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1095 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
1100 NVT,
N->getOperand(IsStrict ? 1 : 0));
1101 TargetLowering::MakeLibCallOptions CallOptions;
1104 std::pair<SDValue, SDValue> Tmp =
1105 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
1106 Op, CallOptions, dl, Chain);
1109 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1115 ReplaceValueWith(
SDValue(
N, 0), TLI.expandVecReduce(
N, DAG));
1119SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
1120 ReplaceValueWith(
SDValue(
N, 0), TLI.expandVecReduceSeq(
N, DAG));
1128bool DAGTypeLegalizer::SoftenFloatOperand(
SDNode *
N,
unsigned OpNo) {
1129 LLVM_DEBUG(
dbgs() <<
"Soften float operand " << OpNo <<
": ";
N->dump(&DAG));
1132 switch (
N->getOpcode()) {
1135 dbgs() <<
"SoftenFloatOperand Op #" << OpNo <<
": ";
1136 N->dump(&DAG);
dbgs() <<
"\n";
1140 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(
N);
break;
1141 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(
N);
break;
1142 case ISD::STRICT_FP_TO_FP16:
1143 case ISD::FP_TO_FP16:
1144 case ISD::FP_TO_BF16:
1145 case ISD::STRICT_FP_TO_BF16:
1154 Res = SoftenFloatOp_FP_TO_XINT_SAT(
N);
break;
1156 case ISD::LROUND: Res = SoftenFloatOp_LROUND(
N);
break;
1158 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(
N);
break;
1160 case ISD::LRINT: Res = SoftenFloatOp_LRINT(
N);
break;
1162 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(
N);
break;
1166 case ISD::SETCC: Res = SoftenFloatOp_SETCC(
N);
break;
1167 case ISD::STORE: Res = SoftenFloatOp_STORE(
N, OpNo);
break;
1168 case ISD::ATOMIC_STORE:
1169 Res = SoftenFloatOp_ATOMIC_STORE(
N, OpNo);
1173 Res = SoftenFloatOp_FAKE_USE(
N);
1178 if (!Res.
getNode())
return false;
1186 "Invalid operand softening");
1188 ReplaceValueWith(
SDValue(
N, 0), Res);
1193 SDValue Op0 = GetSoftenedFloat(
N->getOperand(0));
1195 return DAG.getNode(ISD::BITCAST, SDLoc(
N),
N->getValueType(0), Op0);
1202 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1203 N->getOpcode() == ISD::FP_TO_BF16 ||
1204 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1207 bool IsStrict =
N->isStrictFPOpcode();
1208 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
1209 EVT SVT =
Op.getValueType();
1210 EVT RVT =
N->getValueType(0);
1212 if (
N->getOpcode() == ISD::FP_TO_FP16 ||
1213 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1214 FloatRVT = MVT::f16;
1215 else if (
N->getOpcode() == ISD::FP_TO_BF16 ||
1216 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1217 FloatRVT = MVT::bf16;
1220 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND libcall");
1223 Op = GetSoftenedFloat(
Op);
1224 TargetLowering::MakeLibCallOptions CallOptions;
1226 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT,
Op,
1227 CallOptions, SDLoc(
N),
1230 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1231 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1242 NewLHS = GetSoftenedFloat(NewLHS);
1243 NewRHS = GetSoftenedFloat(NewRHS);
1244 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(
N),
1245 N->getOperand(2),
N->getOperand(3));
1249 if (!NewRHS.getNode()) {
1250 NewRHS = DAG.getConstant(0, SDLoc(
N), NewLHS.
getValueType());
1255 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
1256 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1266 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1267 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1268 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1272 if (Promoted.
bitsGE(RetVT))
1280 bool IsStrict =
N->isStrictFPOpcode();
1284 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
1285 EVT SVT =
Op.getValueType();
1286 EVT RVT =
N->getValueType(0);
1296 "Unsupported FP_TO_XINT!");
1298 Op = GetSoftenedFloat(
Op);
1300 TargetLowering::MakeLibCallOptions CallOptions;
1302 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT,
Op,
1303 CallOptions, dl, Chain);
1311 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1312 ReplaceValueWith(
SDValue(
N, 0), Res);
1316SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(
SDNode *
N) {
1317 SDValue Res = TLI.expandFP_TO_INT_SAT(
N, DAG);
1326 NewLHS = GetSoftenedFloat(NewLHS);
1327 NewRHS = GetSoftenedFloat(NewRHS);
1328 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(
N),
1329 N->getOperand(0),
N->getOperand(1));
1333 if (!NewRHS.getNode()) {
1334 NewRHS = DAG.getConstant(0, SDLoc(
N), NewLHS.
getValueType());
1339 return SDValue(DAG.UpdateNodeOperands(
N, NewLHS, NewRHS,
1340 N->getOperand(2),
N->getOperand(3),
1341 DAG.getCondCode(CCCode)),
1346 bool IsStrict =
N->isStrictFPOpcode();
1354 SDValue NewLHS = GetSoftenedFloat(Op0);
1355 SDValue NewRHS = GetSoftenedFloat(Op1);
1356 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(
N), Op0, Op1,
1363 NewRHS, DAG.getCondCode(CCCode));
1365 return SDValue(DAG.UpdateNodeOperands(
N, NewLHS, NewRHS,
1366 DAG.getCondCode(CCCode)), 0);
1371 "Unexpected setcc expansion!");
1374 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
1375 ReplaceValueWith(
SDValue(
N, 1), Chain);
1381SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
1383 assert(OpNo == 1 &&
"Can only soften the stored value!");
1388 if (
ST->isTruncatingStore())
1390 Val = BitConvertToInteger(
1392 DAG.getIntPtrConstant(0, dl,
true)));
1394 Val = GetSoftenedFloat(Val);
1396 return DAG.getStore(
ST->getChain(), dl, Val,
ST->getBasePtr(),
1397 ST->getMemOperand());
1400SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(
SDNode *
N,
unsigned OpNo) {
1401 assert(OpNo == 1 &&
"Can only soften the stored value!");
1407 assert(
ST->getMemoryVT() == VT &&
"truncating atomic store not handled");
1409 SDValue NewVal = GetSoftenedFloat(Val);
1410 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
ST->getChain(), NewVal,
1411 ST->getBasePtr(),
ST->getMemOperand());
1416 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
1419 EVT LVT =
LHS.getValueType();
1421 EVT RVT =
RHS.getValueType();
1427 int SizeDiff = RSize - LSize;
1431 DAG.getConstant(SizeDiff, dl,
1432 TLI.getShiftAmountTy(
RHS.getValueType(),
1433 DAG.getDataLayout())));
1435 }
else if (SizeDiff < 0) {
1439 DAG.getConstant(-SizeDiff, dl,
1440 TLI.getShiftAmountTy(
RHS.getValueType(),
1441 DAG.getDataLayout())));
1444 RHS = DAG.getBitcast(LVT,
RHS);
1448SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(
SDNode *
N, RTLIB::Libcall LC) {
1449 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1450 bool IsStrict =
N->isStrictFPOpcode();
1451 unsigned Offset = IsStrict ? 1 : 0;
1454 TargetLowering::MakeLibCallOptions CallOptions;
1455 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
1457 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT,
Op,
1458 CallOptions, SDLoc(
N),
1461 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1462 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1470 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1476 RTLIB::LROUND_PPCF128));
1480 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1485 RTLIB::LLROUND_F128,
1486 RTLIB::LLROUND_PPCF128));
1490 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1496 RTLIB::LRINT_PPCF128));
1500 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1506 RTLIB::LLRINT_PPCF128));
1510 SDValue Op1 = BitConvertToInteger(
N->getOperand(1));
1511 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getValueType(0),
1512 N->getOperand(0), Op1);
1523void DAGTypeLegalizer::ExpandFloatResult(
SDNode *
N,
unsigned ResNo) {
1529 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
1532 switch (
N->getOpcode()) {
1535 dbgs() <<
"ExpandFloatResult #" << ResNo <<
": ";
1536 N->dump(&DAG);
dbgs() <<
"\n";
1547 case ISD::BITCAST: ExpandRes_BITCAST(
N,
Lo,
Hi);
break;
1551 case ISD::VAARG: ExpandRes_VAARG(
N,
Lo,
Hi);
break;
1555 case ISD::FABS: ExpandFloatRes_FABS(
N,
Lo,
Hi);
break;
1557 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(
N,
Lo,
Hi);
break;
1559 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(
N,
Lo,
Hi);
break;
1560 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(
N,
Lo,
Hi);
break;
1561 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(
N,
Lo,
Hi);
break;
1565 case ISD::FACOS: ExpandFloatRes_FACOS(
N,
Lo,
Hi);
break;
1567 case ISD::FASIN: ExpandFloatRes_FASIN(
N,
Lo,
Hi);
break;
1569 case ISD::FATAN: ExpandFloatRes_FATAN(
N,
Lo,
Hi);
break;
1571 case ISD::FATAN2: ExpandFloatRes_FATAN2(
N,
Lo,
Hi);
break;
1572 case ISD::FCBRT: ExpandFloatRes_FCBRT(
N,
Lo,
Hi);
break;
1574 case ISD::FCEIL: ExpandFloatRes_FCEIL(
N,
Lo,
Hi);
break;
1577 case ISD::FCOS: ExpandFloatRes_FCOS(
N,
Lo,
Hi);
break;
1579 case ISD::FCOSH: ExpandFloatRes_FCOSH(
N,
Lo,
Hi);
break;
1583 case ISD::FEXP: ExpandFloatRes_FEXP(
N,
Lo,
Hi);
break;
1585 case ISD::FEXP2: ExpandFloatRes_FEXP2(
N,
Lo,
Hi);
break;
1586 case ISD::FEXP10: ExpandFloatRes_FEXP10(
N,
Lo,
Hi);
break;
1588 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(
N,
Lo,
Hi);
break;
1590 case ISD::FLOG: ExpandFloatRes_FLOG(
N,
Lo,
Hi);
break;
1592 case ISD::FLOG2: ExpandFloatRes_FLOG2(
N,
Lo,
Hi);
break;
1594 case ISD::FLOG10: ExpandFloatRes_FLOG10(
N,
Lo,
Hi);
break;
1600 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(
N,
Lo,
Hi);
break;
1601 case ISD::FNEG: ExpandFloatRes_FNEG(
N,
Lo,
Hi);
break;
1603 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(
N,
Lo,
Hi);
break;
1605 case ISD::FPOW: ExpandFloatRes_FPOW(
N,
Lo,
Hi);
break;
1607 case ISD::FPOWI: ExpandFloatRes_FPOWI(
N,
Lo,
Hi);
break;
1612 case ISD::FRINT: ExpandFloatRes_FRINT(
N,
Lo,
Hi);
break;
1614 case ISD::FROUND: ExpandFloatRes_FROUND(
N,
Lo,
Hi);
break;
1616 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(
N,
Lo,
Hi);
break;
1618 case ISD::FSIN: ExpandFloatRes_FSIN(
N,
Lo,
Hi);
break;
1620 case ISD::FSINH: ExpandFloatRes_FSINH(
N,
Lo,
Hi);
break;
1622 case ISD::FSQRT: ExpandFloatRes_FSQRT(
N,
Lo,
Hi);
break;
1626 case ISD::FTAN: ExpandFloatRes_FTAN(
N,
Lo,
Hi);
break;
1628 case ISD::FTANH: ExpandFloatRes_FTANH(
N,
Lo,
Hi);
break;
1630 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(
N,
Lo,
Hi);
break;
1631 case ISD::LOAD: ExpandFloatRes_LOAD(
N,
Lo,
Hi);
break;
1638 case ISD::FMODF: ExpandFloatRes_FMODF(
N);
break;
1639 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(
N);
break;
1640 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(
N);
break;
1651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1653 "Do not know how to expand this float constant!");
1657 Lo = DAG.getConstantFP(
APFloat(Sem,
C.extractBits(64, 64)), dl, NVT);
1658 Hi = DAG.getConstantFP(
APFloat(Sem,
C.extractBits(64, 0)), dl, NVT);
1661void DAGTypeLegalizer::ExpandFloatRes_Unary(
SDNode *
N, RTLIB::Libcall LC,
1663 bool IsStrict =
N->isStrictFPOpcode();
1664 unsigned Offset = IsStrict ? 1 : 0;
1667 TargetLowering::MakeLibCallOptions CallOptions;
1668 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC,
N->getValueType(0),
1669 Op, CallOptions, SDLoc(
N),
1672 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1673 GetPairElements(Tmp.first,
Lo,
Hi);
1676void DAGTypeLegalizer::ExpandFloatRes_Binary(
SDNode *
N, RTLIB::Libcall LC,
1678 bool IsStrict =
N->isStrictFPOpcode();
1679 unsigned Offset = IsStrict ? 1 : 0;
1682 TargetLowering::MakeLibCallOptions CallOptions;
1683 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC,
N->getValueType(0),
1684 Ops, CallOptions, SDLoc(
N),
1687 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1688 GetPairElements(Tmp.first,
Lo,
Hi);
1691void DAGTypeLegalizer::ExpandFloatRes_FMODF(
SDNode *
N) {
1692 ExpandFloatRes_UnaryWithTwoFPResults(
N,
RTLIB::getMODF(
N->getValueType(0)),
1696void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(
SDNode *
N) {
1700void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(
SDNode *
N) {
1701 ExpandFloatRes_UnaryWithTwoFPResults(
N,
1705void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1706 SDNode *
N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1707 assert(!
N->isStrictFPOpcode() &&
"strictfp not implemented");
1709 DAG.expandMultipleResultFPLibCall(LC,
N,
Results, CallRetResNo);
1712 GetPairElements(Res,
Lo,
Hi);
1719 assert(
N->getValueType(0) == MVT::ppcf128 &&
1720 "Logic only correct for ppcf128!");
1723 GetExpandedFloat(
N->getOperand(0),
Lo, Tmp);
1726 Lo = DAG.getSelectCC(dl, Tmp,
Hi,
Lo,
1727 DAG.getNode(ISD::FNEG, dl,
Lo.getValueType(),
Lo),
1734 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1735 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1736 RTLIB::FMIN_PPCF128),
Lo,
Hi);
1742 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1743 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1744 RTLIB::FMAX_PPCF128),
Lo,
Hi);
1749 ExpandFloatRes_Binary(
1752 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
1753 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128),
1759 ExpandFloatRes_Binary(
1762 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
1763 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128),
1770 RTLIB::ADD_F32, RTLIB::ADD_F64,
1771 RTLIB::ADD_F80, RTLIB::ADD_F128,
1772 RTLIB::ADD_PPCF128),
Lo,
Hi);
1777 ExpandFloatRes_Unary(
N,
1779 RTLIB::ACOS_F64, RTLIB::ACOS_F80,
1780 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128),
1786 ExpandFloatRes_Unary(
N,
1788 RTLIB::ASIN_F64, RTLIB::ASIN_F80,
1789 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128),
1795 ExpandFloatRes_Unary(
N,
1797 RTLIB::ATAN_F64, RTLIB::ATAN_F80,
1798 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128),
1804 ExpandFloatRes_Binary(
N,
1806 RTLIB::ATAN2_F64, RTLIB::ATAN2_F80,
1807 RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128),
1813 ExpandFloatRes_Unary(
N,
GetFPLibCall(
N->getValueType(0), RTLIB::CBRT_F32,
1814 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1816 RTLIB::CBRT_PPCF128),
Lo,
Hi);
1819void DAGTypeLegalizer::ExpandFloatRes_FCEIL(
SDNode *
N,
1822 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1823 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1824 RTLIB::CEIL_PPCF128),
Lo,
Hi);
1827void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(
SDNode *
N,
1830 RTLIB::COPYSIGN_F32,
1831 RTLIB::COPYSIGN_F64,
1832 RTLIB::COPYSIGN_F80,
1833 RTLIB::COPYSIGN_F128,
1834 RTLIB::COPYSIGN_PPCF128),
Lo,
Hi);
1837void DAGTypeLegalizer::ExpandFloatRes_FCOS(
SDNode *
N,
1840 RTLIB::COS_F32, RTLIB::COS_F64,
1841 RTLIB::COS_F80, RTLIB::COS_F128,
1842 RTLIB::COS_PPCF128),
Lo,
Hi);
1847 ExpandFloatRes_Unary(
N,
1849 RTLIB::COSH_F64, RTLIB::COSH_F80,
1850 RTLIB::COSH_F128, RTLIB::COSH_PPCF128),
1861 RTLIB::DIV_PPCF128),
Lo,
Hi);
1864void DAGTypeLegalizer::ExpandFloatRes_FEXP(
SDNode *
N,
1867 RTLIB::EXP_F32, RTLIB::EXP_F64,
1868 RTLIB::EXP_F80, RTLIB::EXP_F128,
1869 RTLIB::EXP_PPCF128),
Lo,
Hi);
1872void DAGTypeLegalizer::ExpandFloatRes_FEXP2(
SDNode *
N,
1875 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1876 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1877 RTLIB::EXP2_PPCF128),
Lo,
Hi);
1882 ExpandFloatRes_Unary(
N,
1884 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1885 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1889void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(
SDNode *
N,
1892 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1893 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1894 RTLIB::FLOOR_PPCF128),
Lo,
Hi);
1897void DAGTypeLegalizer::ExpandFloatRes_FLOG(
SDNode *
N,
1900 RTLIB::LOG_F32, RTLIB::LOG_F64,
1901 RTLIB::LOG_F80, RTLIB::LOG_F128,
1902 RTLIB::LOG_PPCF128),
Lo,
Hi);
1905void DAGTypeLegalizer::ExpandFloatRes_FLOG2(
SDNode *
N,
1908 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1909 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1910 RTLIB::LOG2_PPCF128),
Lo,
Hi);
1913void DAGTypeLegalizer::ExpandFloatRes_FLOG10(
SDNode *
N,
1916 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1917 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1918 RTLIB::LOG10_PPCF128),
Lo,
Hi);
1923 bool IsStrict =
N->isStrictFPOpcode();
1924 unsigned Offset = IsStrict ? 1 : 0;
1928 TargetLowering::MakeLibCallOptions CallOptions;
1929 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
GetFPLibCall(
N->getValueType(0),
1934 RTLIB::FMA_PPCF128),
1935 N->getValueType(0),
Ops, CallOptions,
1938 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1939 GetPairElements(Tmp.first,
Lo,
Hi);
1949 RTLIB::MUL_PPCF128),
Lo,
Hi);
1952void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(
SDNode *
N,
1955 RTLIB::NEARBYINT_F32,
1956 RTLIB::NEARBYINT_F64,
1957 RTLIB::NEARBYINT_F80,
1958 RTLIB::NEARBYINT_F128,
1959 RTLIB::NEARBYINT_PPCF128),
Lo,
Hi);
1965 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1966 Lo = DAG.getNode(ISD::FNEG, dl,
Lo.getValueType(),
Lo);
1967 Hi = DAG.getNode(ISD::FNEG, dl,
Hi.getValueType(),
Hi);
1970void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(
SDNode *
N,
SDValue &
Lo,
1974 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1979 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1981 bool IsStrict =
N->isStrictFPOpcode();
1986 if (NVT ==
N->getOperand(1).getValueType()) {
1987 Hi =
N->getOperand(1);
1992 {
N->getOperand(0),
N->getOperand(1) });
1996 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT,
N->getOperand(0));
2002 ReplaceValueWith(
SDValue(
N, 1), Chain);
2005void DAGTypeLegalizer::ExpandFloatRes_FPOW(
SDNode *
N,
2008 RTLIB::POW_F32, RTLIB::POW_F64,
2009 RTLIB::POW_F80, RTLIB::POW_F128,
2010 RTLIB::POW_PPCF128),
Lo,
Hi);
2013void DAGTypeLegalizer::ExpandFloatRes_FPOWI(
SDNode *
N,
2023void DAGTypeLegalizer::ExpandFloatRes_FREEZE(
SDNode *
N,
2025 assert(
N->getValueType(0) == MVT::ppcf128 &&
2026 "Logic only correct for ppcf128!");
2029 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
2034void DAGTypeLegalizer::ExpandFloatRes_FREM(
SDNode *
N,
2037 RTLIB::REM_F32, RTLIB::REM_F64,
2038 RTLIB::REM_F80, RTLIB::REM_F128,
2039 RTLIB::REM_PPCF128),
Lo,
Hi);
2042void DAGTypeLegalizer::ExpandFloatRes_FRINT(
SDNode *
N,
2045 RTLIB::RINT_F32, RTLIB::RINT_F64,
2046 RTLIB::RINT_F80, RTLIB::RINT_F128,
2047 RTLIB::RINT_PPCF128),
Lo,
Hi);
2050void DAGTypeLegalizer::ExpandFloatRes_FROUND(
SDNode *
N,
2057 RTLIB::ROUND_PPCF128),
Lo,
Hi);
2060void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(
SDNode *
N,
2063 RTLIB::ROUNDEVEN_F32,
2064 RTLIB::ROUNDEVEN_F64,
2065 RTLIB::ROUNDEVEN_F80,
2066 RTLIB::ROUNDEVEN_F128,
2067 RTLIB::ROUNDEVEN_PPCF128),
Lo,
Hi);
2070void DAGTypeLegalizer::ExpandFloatRes_FSIN(
SDNode *
N,
2073 RTLIB::SIN_F32, RTLIB::SIN_F64,
2074 RTLIB::SIN_F80, RTLIB::SIN_F128,
2075 RTLIB::SIN_PPCF128),
Lo,
Hi);
2080 ExpandFloatRes_Unary(
N,
2082 RTLIB::SINH_F64, RTLIB::SINH_F80,
2083 RTLIB::SINH_F128, RTLIB::SINH_PPCF128),
2087void DAGTypeLegalizer::ExpandFloatRes_FSQRT(
SDNode *
N,
2090 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
2091 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
2092 RTLIB::SQRT_PPCF128),
Lo,
Hi);
2102 RTLIB::SUB_PPCF128),
Lo,
Hi);
2107 ExpandFloatRes_Unary(
N,
2109 RTLIB::TAN_F64, RTLIB::TAN_F80,
2110 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
2116 ExpandFloatRes_Unary(
N,
2118 RTLIB::TANH_F64, RTLIB::TANH_F80,
2119 RTLIB::TANH_F128, RTLIB::TANH_PPCF128),
2123void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(
SDNode *
N,
2126 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
2127 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
2128 RTLIB::TRUNC_PPCF128),
Lo,
Hi);
2134 ExpandRes_NormalLoad(
N,
Lo,
Hi);
2144 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
LD->getValueType(0));
2146 assert(
LD->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
2148 Hi = DAG.getExtLoad(
LD->getExtensionType(), dl, NVT, Chain,
Ptr,
2149 LD->getMemoryVT(),
LD->getMemOperand());
2159 ReplaceValueWith(
SDValue(LD, 1), Chain);
2164 assert(
N->getValueType(0) == MVT::ppcf128 &&
"Unsupported XINT_TO_FP!");
2165 EVT VT =
N->getValueType(0);
2166 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2167 bool Strict =
N->isStrictFPOpcode();
2168 SDValue Src =
N->getOperand(Strict ? 1 : 0);
2169 EVT SrcVT = Src.getValueType();
2177 Flags.setNoFPExcept(
N->getFlags().hasNoFPExcept());
2182 if (SrcVT.
bitsLE(MVT::i32)) {
2186 Hi = DAG.getNode(
N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
2187 {Chain, Src}, Flags);
2190 Hi = DAG.getNode(
N->getOpcode(), dl, NVT, Src);
2192 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2193 if (SrcVT.
bitsLE(MVT::i64)) {
2196 LC = RTLIB::SINTTOFP_I64_PPCF128;
2197 }
else if (SrcVT.
bitsLE(MVT::i128)) {
2199 LC = RTLIB::SINTTOFP_I128_PPCF128;
2201 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
2203 TargetLowering::MakeLibCallOptions CallOptions;
2205 std::pair<SDValue, SDValue> Tmp =
2206 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
2209 GetPairElements(Tmp.first,
Lo,
Hi);
2215 ReplaceValueWith(
SDValue(
N, 1), Chain);
2225 SrcVT = Src.getValueType();
2228 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2229 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2230 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2231 ArrayRef<uint64_t> Parts;
2248 SDValue NewLo = DAG.getConstantFP(
2252 {Chain, Hi, NewLo}, Flags);
2254 ReplaceValueWith(
SDValue(
N, 1), Chain);
2257 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
2259 GetPairElements(
Lo,
Lo,
Hi);
2271bool DAGTypeLegalizer::ExpandFloatOperand(
SDNode *
N,
unsigned OpNo) {
2276 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
2279 switch (
N->getOpcode()) {
2282 dbgs() <<
"ExpandFloatOperand Op #" << OpNo <<
": ";
2283 N->dump(&DAG);
dbgs() <<
"\n";
2287 case ISD::BITCAST: Res = ExpandOp_BITCAST(
N);
break;
2291 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(
N);
break;
2299 case ISD::LROUND: Res = ExpandFloatOp_LROUND(
N);
break;
2300 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(
N);
break;
2301 case ISD::LRINT: Res = ExpandFloatOp_LRINT(
N);
break;
2302 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(
N);
break;
2306 case ISD::SETCC: Res = ExpandFloatOp_SETCC(
N);
break;
2312 if (!Res.
getNode())
return false;
2320 "Invalid operand expansion");
2322 ReplaceValueWith(
SDValue(
N, 0), Res);
2328void DAGTypeLegalizer::FloatExpandSetCCOperands(
SDValue &NewLHS,
2333 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2334 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2335 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2344 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2345 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.
getValueType()), LHSHi,
2348 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.
getValueType()), LHSLo,
2349 RHSLo, CCCode, OutputChain, IsSignaling);
2353 DAG.getSetCC(dl, getSetCCResultType(LHSHi.
getValueType()), LHSHi, RHSHi,
2356 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.
getValueType()), LHSHi,
2357 RHSHi, CCCode, OutputChain, IsSignaling);
2362 Chain = OutputChain;
2369 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(
N), Chain);
2374 NewRHS = DAG.getConstant(0, SDLoc(
N), NewLHS.
getValueType());
2379 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
2380 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2381 N->getOperand(4)), 0);
2385 assert(
N->getOperand(1).getValueType() == MVT::ppcf128 &&
2386 "Logic only correct for ppcf128!");
2388 GetExpandedFloat(
N->getOperand(1),
Lo,
Hi);
2392 N->getValueType(0),
N->getOperand(0),
Hi);
2396 bool IsStrict =
N->isStrictFPOpcode();
2397 assert(
N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2398 "Logic only correct for ppcf128!");
2400 GetExpandedFloat(
N->getOperand(IsStrict ? 1 : 0),
Lo,
Hi);
2405 N->getValueType(0),
Hi,
N->getOperand(1));
2409 if (
Hi.getValueType() ==
N->getValueType(0)) {
2411 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
2417 {
N->getValueType(0), MVT::Other},
2418 {
N->getOperand(0),
Hi,
N->getOperand(2)});
2425 EVT RVT =
N->getValueType(0);
2428 bool IsStrict =
N->isStrictFPOpcode();
2431 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
2437 "Unsupported FP_TO_XINT!");
2438 TargetLowering::MakeLibCallOptions CallOptions;
2439 std::pair<SDValue, SDValue> Tmp =
2440 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
2444 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2445 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2453 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(
N), Chain);
2458 NewRHS = DAG.getConstant(0, SDLoc(
N), NewLHS.
getValueType());
2463 return SDValue(DAG.UpdateNodeOperands(
N, NewLHS, NewRHS,
2464 N->getOperand(2),
N->getOperand(3),
2465 DAG.getCondCode(CCCode)), 0);
2469 bool IsStrict =
N->isStrictFPOpcode();
2475 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(
N), Chain,
2481 "Unexpected setcc expansion!");
2483 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
2484 ReplaceValueWith(
SDValue(
N, 1), Chain);
2490SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2492 return ExpandOp_NormalStore(
N, OpNo);
2495 assert(OpNo == 1 &&
"Can only expand the stored value so far");
2501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2502 ST->getValue().getValueType());
2504 assert(
ST->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
2508 GetExpandedOp(
ST->getValue(),
Lo,
Hi);
2510 return DAG.getTruncStore(Chain, SDLoc(
N),
Hi,
Ptr,
2511 ST->getMemoryVT(),
ST->getMemOperand());
2515 EVT RVT =
N->getValueType(0);
2516 EVT RetVT =
N->getOperand(0).getValueType();
2517 TargetLowering::MakeLibCallOptions CallOptions;
2523 RTLIB::LROUND_PPCF128),
2524 RVT,
N->getOperand(0), CallOptions, SDLoc(
N)).first;
2528 EVT RVT =
N->getValueType(0);
2529 EVT RetVT =
N->getOperand(0).getValueType();
2530 TargetLowering::MakeLibCallOptions CallOptions;
2535 RTLIB::LLROUND_F128,
2536 RTLIB::LLROUND_PPCF128),
2537 RVT,
N->getOperand(0), CallOptions, SDLoc(
N)).first;
2541 EVT RVT =
N->getValueType(0);
2542 EVT RetVT =
N->getOperand(0).getValueType();
2543 TargetLowering::MakeLibCallOptions CallOptions;
2549 RTLIB::LRINT_PPCF128),
2550 RVT,
N->getOperand(0), CallOptions, SDLoc(
N)).first;
2554 EVT RVT =
N->getValueType(0);
2555 EVT RetVT =
N->getOperand(0).getValueType();
2556 TargetLowering::MakeLibCallOptions CallOptions;
2562 RTLIB::LLRINT_PPCF128),
2563 RVT,
N->getOperand(0), CallOptions, SDLoc(
N)).first;
2572 if (OpVT == MVT::f16)
2573 return ISD::FP16_TO_FP;
2574 if (RetVT == MVT::f16)
2575 return ISD::FP_TO_FP16;
2576 if (OpVT == MVT::bf16)
2577 return ISD::BF16_TO_FP;
2578 if (RetVT == MVT::bf16)
2579 return ISD::FP_TO_BF16;
2584 if (OpVT == MVT::f16)
2585 return ISD::STRICT_FP16_TO_FP;
2586 if (RetVT == MVT::f16)
2587 return ISD::STRICT_FP_TO_FP16;
2588 if (OpVT == MVT::bf16)
2589 return ISD::STRICT_BF16_TO_FP;
2590 if (RetVT == MVT::bf16)
2591 return ISD::STRICT_FP_TO_BF16;
2595bool DAGTypeLegalizer::PromoteFloatOperand(
SDNode *
N,
unsigned OpNo) {
2596 LLVM_DEBUG(
dbgs() <<
"Promote float operand " << OpNo <<
": ";
N->dump(&DAG));
2599 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
2610 switch (
N->getOpcode()) {
2613 dbgs() <<
"PromoteFloatOperand Op #" << OpNo <<
": ";
2614 N->dump(&DAG);
dbgs() <<
"\n";
2618 case ISD::BITCAST:
R = PromoteFloatOp_BITCAST(
N, OpNo);
break;
2620 R = PromoteFloatOp_FAKE_USE(
N, OpNo);
2628 case ISD::LLRINT:
R = PromoteFloatOp_UnaryOp(
N, OpNo);
break;
2632 R = PromoteFloatOp_FP_TO_XINT_SAT(
N, OpNo);
break;
2633 case ISD::FP_EXTEND:
R = PromoteFloatOp_FP_EXTEND(
N, OpNo);
break;
2635 R = PromoteFloatOp_STRICT_FP_EXTEND(
N, OpNo);
2638 case ISD::SETCC:
R = PromoteFloatOp_SETCC(
N, OpNo);
break;
2639 case ISD::STORE:
R = PromoteFloatOp_STORE(
N, OpNo);
break;
2640 case ISD::ATOMIC_STORE:
R = PromoteFloatOp_ATOMIC_STORE(
N, OpNo);
break;
2645 ReplaceValueWith(
SDValue(
N, 0), R);
2649SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(
SDNode *
N,
unsigned OpNo) {
2651 EVT OpVT =
Op->getValueType(0);
2653 SDValue Promoted = GetPromotedFloat(
N->getOperand(0));
2662 return DAG.getBitcast(
N->getValueType(0), Convert);
2665SDValue DAGTypeLegalizer::PromoteFloatOp_FAKE_USE(
SDNode *
N,
unsigned OpNo) {
2666 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
2667 SDValue Op = GetPromotedFloat(
N->getOperand(OpNo));
2668 return DAG.getNode(
N->getOpcode(), SDLoc(
N), MVT::Other,
N->getOperand(0),
2674SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(
SDNode *
N,
unsigned OpNo) {
2675 assert (OpNo == 1 &&
"Only Operand 1 must need promotion here");
2676 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2678 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getValueType(0),
2679 N->getOperand(0), Op1);
2683SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(
SDNode *
N,
unsigned OpNo) {
2684 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2685 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getValueType(0),
Op);
2689SDValue DAGTypeLegalizer::PromoteFloatOp_AssertNoFPClass(
SDNode *
N,
2691 return GetPromotedFloat(
N->getOperand(0));
2694SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(
SDNode *
N,
2696 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2697 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getValueType(0),
Op,
2701SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(
SDNode *
N,
unsigned OpNo) {
2702 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2703 EVT VT =
N->getValueType(0);
2706 if (VT ==
Op->getValueType(0))
2710 return DAG.getNode(ISD::FP_EXTEND, SDLoc(
N), VT,
Op);
2713SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(
SDNode *
N,
2715 assert(OpNo == 1 &&
"Promoting unpromotable operand");
2717 SDValue Op = GetPromotedFloat(
N->getOperand(1));
2718 EVT VT =
N->getValueType(0);
2721 if (VT ==
Op->getValueType(0)) {
2722 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
2728 N->getOperand(0),
Op);
2736SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2741 LHS,
RHS,
N->getOperand(2),
N->getOperand(3),
2747SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2748 EVT VT =
N->getValueType(0);
2749 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2750 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2753 return DAG.getSetCC(SDLoc(
N), VT, Op0, Op1, CCCode);
2759SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2764 SDValue Promoted = GetPromotedFloat(Val);
2765 EVT VT =
ST->getOperand(1).getValueType();
2772 return DAG.getStore(
ST->getChain(),
DL, NewVal,
ST->getBasePtr(),
2773 ST->getMemOperand());
2782 SDValue Promoted = GetPromotedFloat(Val);
2783 EVT VT =
ST->getOperand(1).getValueType();
2789 return DAG.getAtomic(ISD::ATOMIC_STORE,
DL, IVT,
ST->getChain(), NewVal,
2790 ST->getBasePtr(),
ST->getMemOperand());
2797void DAGTypeLegalizer::PromoteFloatResult(
SDNode *
N,
unsigned ResNo) {
2798 LLVM_DEBUG(
dbgs() <<
"Promote float result " << ResNo <<
": ";
N->dump(&DAG));
2802 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
2807 switch (
N->getOpcode()) {
2810 case ISD::FP16_TO_FP:
2811 case ISD::FP_TO_FP16:
2814 dbgs() <<
"PromoteFloatResult #" << ResNo <<
": ";
2815 N->dump(&DAG);
dbgs() <<
"\n";
2820 R = PromoteFloatRes_BITCAST(
N);
2823 R = PromoteFloatRes_FREEZE(
N);
2827 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(
N);
break;
2846 case ISD::FNEARBYINT:
2850 case ISD::FROUNDEVEN:
2859 R = PromoteFloatRes_AssertNoFPClass(
N);
2867 case ISD::FMAXIMUMNUM:
2868 case ISD::FMINIMUMNUM:
2871 case ISD::FMAXNUM_IEEE:
2872 case ISD::FMINNUM_IEEE:
2877 case ISD::FSUB:
R = PromoteFloatRes_BinOp(
N);
break;
2880 case ISD::FMAD:
R = PromoteFloatRes_FMAD(
N);
break;
2883 case ISD::FLDEXP:
R = PromoteFloatRes_ExpOp(
N);
break;
2884 case ISD::FFREXP:
R = PromoteFloatRes_FFREXP(
N);
break;
2888 case ISD::FSINCOSPI:
2889 R = PromoteFloatRes_UnaryWithTwoFPResults(
N);
2893 R = PromoteFloatRes_STRICT_FP_ROUND(
N);
2895 case ISD::LOAD:
R = PromoteFloatRes_LOAD(
N);
break;
2896 case ISD::ATOMIC_LOAD:
2897 R = PromoteFloatRes_ATOMIC_LOAD(
N);
2906 case ISD::ATOMIC_SWAP:
R = BitcastToInt_ATOMIC_SWAP(
N);
break;
2907 case ISD::VECREDUCE_FADD:
2908 case ISD::VECREDUCE_FMUL:
2909 case ISD::VECREDUCE_FMIN:
2910 case ISD::VECREDUCE_FMAX:
2911 case ISD::VECREDUCE_FMAXIMUM:
2912 case ISD::VECREDUCE_FMINIMUM:
2913 R = PromoteFloatRes_VECREDUCE(
N);
2915 case ISD::VECREDUCE_SEQ_FADD:
2916 case ISD::VECREDUCE_SEQ_FMUL:
2917 R = PromoteFloatRes_VECREDUCE_SEQ(
N);
2922 SetPromotedFloat(
SDValue(
N, ResNo), R);
2931 EVT VT =
N->getValueType(0);
2932 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2936 N->getOperand(0).getValueType().getSizeInBits());
2937 SDValue Cast = DAG.getBitcast(IVT,
N->getOperand(0));
2942 EVT VT =
N->getValueType(0);
2943 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2947 N->getOperand(0).getValueType().getSizeInBits());
2948 SDValue Cast = DAG.getBitcast(IVT,
N->getOperand(0));
2950 DAG.getFreeze(Cast));
2955 EVT VT =
N->getValueType(0);
2966 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2974SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
2987 switch (getTypeAction(VecVT)) {
2990 SDValue Res = GetScalarizedVector(
N->getOperand(0));
2991 ReplaceValueWith(
SDValue(
N, 0), Res);
2995 Vec = GetWidenedVector(Vec);
2997 ReplaceValueWith(
SDValue(
N, 0), Res);
3002 GetSplitVector(Vec,
Lo,
Hi);
3004 uint64_t LoElts =
Lo.getValueType().getVectorNumElements();
3006 if (IdxVal < LoElts)
3010 DAG.getConstant(IdxVal - LoElts,
DL,
3012 ReplaceValueWith(
SDValue(
N, 0), Res);
3020 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
3025 NewOp,
N->getOperand(1));
3028 EVT VT =
N->getValueType(0);
3029 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3037 EVT VT =
N->getValueType(0);
3038 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3039 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3043 return DAG.
getNode(
N->getOpcode(), SDLoc(
N), NVT, Op0, Op1);
3050 EVT VT =
N->getValueType(0);
3051 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3052 SDValue Op = GetPromotedFloat(
N->getOperand(0));
3053 return DAG.getNode(
N->getOpcode(), SDLoc(
N), NVT,
Op);
3059SDValue DAGTypeLegalizer::PromoteFloatRes_AssertNoFPClass(
SDNode *
N) {
3060 return GetPromotedFloat(
N->getOperand(0));
3067 EVT VT =
N->getValueType(0);
3068 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3069 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3070 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
3071 return DAG.getNode(
N->getOpcode(), SDLoc(
N), NVT, Op0, Op1,
N->getFlags());
3075 EVT VT =
N->getValueType(0);
3076 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3077 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3078 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
3079 SDValue Op2 = GetPromotedFloat(
N->getOperand(2));
3081 return DAG.getNode(
N->getOpcode(), SDLoc(
N), NVT, Op0, Op1, Op2);
3086 EVT VT =
N->getValueType(0);
3087 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3088 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
3091 return DAG.
getNode(
N->getOpcode(), SDLoc(
N), NVT, Op0, Op1);
3095 EVT VT =
N->getValueType(0);
3096 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3097 SDValue Op = GetPromotedFloat(
N->getOperand(0));
3099 DAG.
getNode(
N->getOpcode(), SDLoc(
N), {NVT,
N->getValueType(1)},
Op);
3105SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryWithTwoFPResults(
SDNode *
N) {
3106 EVT VT =
N->getValueType(0);
3107 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3108 SDValue Op = GetPromotedFloat(
N->getOperand(0));
3111 for (
unsigned ResNum = 0, NumValues =
N->getNumValues(); ResNum < NumValues;
3125 EVT VT =
N->getValueType(0);
3126 EVT OpVT =
Op->getValueType(0);
3127 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
3138SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(
SDNode *
N) {
3143 EVT VT =
N->getValueType(0);
3144 EVT OpVT =
Op->getValueType(0);
3145 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
3150 DAG.getVTList(IVT, MVT::Other), Chain,
Op);
3154 DAG.getVTList(NVT, MVT::Other), Round.
getValue(1), Round);
3161 EVT VT =
N->getValueType(0);
3166 L->getAddressingMode(),
L->getExtensionType(), IVT, SDLoc(
N),
3167 L->getChain(),
L->getBasePtr(),
L->getOffset(),
L->getPointerInfo(), IVT,
3168 L->getBaseAlign(),
L->getMemOperand()->getFlags(),
L->getAAInfo());
3174 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3178SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(
SDNode *
N) {
3185 ISD::ATOMIC_LOAD, SDLoc(
N), IVT, DAG.getVTList(IVT, MVT::Other),
3193 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3203 N->getOperand(0), TrueVal, FalseVal);
3213 TrueVal.getNode()->getValueType(0),
N->getOperand(0),
3214 N->getOperand(1), TrueVal, FalseVal,
N->getOperand(4));
3221 EVT VT =
N->getValueType(0);
3222 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3223 SDValue NV = DAG.getNode(
N->getOpcode(),
DL, NVT,
N->getOperand(0));
3226 ISD::FP_EXTEND,
DL, NVT,
3228 DAG.getIntPtrConstant(0,
DL,
true)));
3232 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
3233 N->getValueType(0)));
3241 ReplaceValueWith(
SDValue(
N, 0), TLI.expandVecReduce(
N, DAG));
3245SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
3246 ReplaceValueWith(
SDValue(
N, 0), TLI.expandVecReduceSeq(
N, DAG));
3251 EVT VT =
N->getValueType(0);
3260 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
3261 DAG.getVTList(CastVT, MVT::Other),
3262 { AM->getChain(), AM->getBasePtr(), CastVal },
3268 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3285void DAGTypeLegalizer::SoftPromoteHalfResult(
SDNode *
N,
unsigned ResNo) {
3286 LLVM_DEBUG(
dbgs() <<
"Soft promote half result " << ResNo <<
": ";
3291 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
3296 switch (
N->getOpcode()) {
3299 dbgs() <<
"SoftPromoteHalfResult #" << ResNo <<
": ";
3300 N->dump(&DAG);
dbgs() <<
"\n";
3305 case ISD::ARITH_FENCE:
3306 R = SoftPromoteHalfRes_ARITH_FENCE(
N);
break;
3307 case ISD::BITCAST:
R = SoftPromoteHalfRes_BITCAST(
N);
break;
3310 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
N);
break;
3331 case ISD::FNEARBYINT:
3336 case ISD::FROUNDEVEN:
3345 R = SoftPromoteHalfRes_AssertNoFPClass(
N);
3353 case ISD::FMAXIMUMNUM:
3354 case ISD::FMINIMUMNUM:
3361 case ISD::FSUB:
R = SoftPromoteHalfRes_BinOp(
N);
break;
3364 case ISD::FMAD:
R = SoftPromoteHalfRes_FMAD(
N);
break;
3367 case ISD::FLDEXP:
R = SoftPromoteHalfRes_ExpOp(
N);
break;
3369 case ISD::FFREXP:
R = SoftPromoteHalfRes_FFREXP(
N);
break;
3373 case ISD::FSINCOSPI:
3374 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(
N);
3377 case ISD::LOAD:
R = SoftPromoteHalfRes_LOAD(
N);
break;
3378 case ISD::ATOMIC_LOAD:
3379 R = SoftPromoteHalfRes_ATOMIC_LOAD(
N);
3388 case ISD::UNDEF:
R = SoftPromoteHalfRes_UNDEF(
N);
break;
3389 case ISD::ATOMIC_SWAP:
R = BitcastToInt_ATOMIC_SWAP(
N);
break;
3390 case ISD::VECREDUCE_FADD:
3391 case ISD::VECREDUCE_FMUL:
3392 case ISD::VECREDUCE_FMIN:
3393 case ISD::VECREDUCE_FMAX:
3394 case ISD::VECREDUCE_FMAXIMUM:
3395 case ISD::VECREDUCE_FMINIMUM:
3396 R = SoftPromoteHalfRes_VECREDUCE(
N);
3398 case ISD::VECREDUCE_SEQ_FADD:
3399 case ISD::VECREDUCE_SEQ_FMUL:
3400 R = SoftPromoteHalfRes_VECREDUCE_SEQ(
N);
3405 SetSoftPromotedHalf(
SDValue(
N, ResNo), R);
3408SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(
SDNode *
N) {
3409 return DAG.
getNode(ISD::ARITH_FENCE, SDLoc(
N), MVT::i16,
3410 BitConvertToInteger(
N->getOperand(0)));
3414 return BitConvertToInteger(
N->getOperand(0));
3417SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(
SDNode *
N) {
3425SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
3426 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
3432SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(
SDNode *
N) {
3433 SDValue LHS = GetSoftPromotedHalf(
N->getOperand(0));
3434 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
3437 EVT LVT =
LHS.getValueType();
3438 EVT RVT =
RHS.getValueType();
3445 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3446 DAG.getConstant(RSize - 1, dl,
3447 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3455 DAG.getConstant(SizeDiff, dl,
3457 DAG.getDataLayout())));
3459 }
else if (SizeDiff < 0) {
3463 DAG.getConstant(-SizeDiff, dl,
3465 DAG.getDataLayout())));
3470 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3471 DAG.getConstant(LSize - 1, dl,
3472 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3473 Mask = DAG.getNode(
ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3477 return DAG.getNode(
ISD::OR, dl, LVT,
LHS, SignBit);
3482 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3483 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3484 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
3485 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
3490 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3491 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3492 Op2 = DAG.
getNode(PromotionOpcode, dl, NVT, Op2);
3501 EVT OVT =
N->getValueType(0);
3502 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3503 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3517 EVT OVT =
N->getValueType(0);
3518 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3519 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
3526 DAG.getVTList(NVT,
N->getValueType(1)),
Op);
3534SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(
SDNode *
N) {
3535 EVT OVT =
N->getValueType(0);
3536 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3537 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
3546 for (
unsigned ResNum = 0, NumValues =
N->getNumValues(); ResNum < NumValues;
3548 SDValue Trunc = DAG.getNode(Truncate, dl, MVT::i16, Res.
getValue(ResNum));
3549 SetSoftPromotedHalf(
SDValue(
N, ResNum), Trunc);
3555SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(
SDNode *
N) {
3556 EVT RVT =
N->getValueType(0);
3557 bool IsStrict =
N->isStrictFPOpcode();
3558 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3559 EVT SVT =
Op.getValueType();
3565 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND libcall");
3568 Op = GetSoftenedFloat(
Op);
3569 TargetLowering::MakeLibCallOptions CallOptions;
3571 std::pair<SDValue, SDValue> Tmp =
3572 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, SDLoc(
N), Chain);
3574 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3575 return DAG.getNode(ISD::BITCAST, SDLoc(
N), MVT::i16, Tmp.first);
3580 {MVT::i16, MVT::Other}, {
N->getOperand(0),
Op});
3595 DAG.getLoad(
L->getAddressingMode(),
L->getExtensionType(), MVT::i16,
3596 SDLoc(
N),
L->getChain(),
L->getBasePtr(),
L->getOffset(),
3597 L->getPointerInfo(), MVT::i16,
L->getBaseAlign(),
3598 L->getMemOperand()->getFlags(),
L->getAAInfo());
3605SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(
SDNode *
N) {
3610 ISD::ATOMIC_LOAD, SDLoc(
N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3620 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
3621 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
3622 return DAG.getSelect(SDLoc(
N), Op1.
getValueType(),
N->getOperand(0), Op1,
3626SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(
SDNode *
N) {
3627 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
3628 SDValue Op3 = GetSoftPromotedHalf(
N->getOperand(3));
3630 N->getOperand(0),
N->getOperand(1), Op2, Op3,
3634SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(
SDNode *
N) {
3635 EVT OVT =
N->getValueType(0);
3636 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3639 if (
N->isStrictFPOpcode()) {
3640 SDValue Op = DAG.getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
3641 {N->getOperand(0), N->getOperand(1)});
3643 {MVT::i16, MVT::Other}, {
Op.getValue(1),
Op});
3644 ReplaceValueWith(
SDValue(
N, 1),
Op.getValue(1));
3655 return DAG.getUNDEF(MVT::i16);
3659 EVT OVT =
N->getValueType(0);
3660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3661 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
3673SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(
SDNode *
N) {
3674 return GetSoftPromotedHalf(
N->getOperand(0));
3678 EVT OVT =
N->getValueType(0);
3679 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3680 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3681 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
3686 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3687 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3695SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(
SDNode *
N) {
3697 ReplaceValueWith(
SDValue(
N, 0), TLI.expandVecReduce(
N, DAG));
3701SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(
SDNode *
N) {
3703 ReplaceValueWith(
SDValue(
N, 0), TLI.expandVecReduceSeq(
N, DAG));
3711bool DAGTypeLegalizer::SoftPromoteHalfOperand(
SDNode *
N,
unsigned OpNo) {
3712 LLVM_DEBUG(
dbgs() <<
"Soft promote half operand " << OpNo <<
": ";
3716 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
3726 switch (
N->getOpcode()) {
3729 dbgs() <<
"SoftPromoteHalfOperand Op #" << OpNo <<
": ";
3730 N->dump(&DAG);
dbgs() <<
"\n";
3735 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(
N);
break;
3737 Res = SoftPromoteHalfOp_FAKE_USE(
N, OpNo);
3739 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(
N, OpNo);
break;
3746 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(
N);
break;
3748 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(
N);
break;
3749 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(
N, OpNo);
break;
3750 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(
N);
break;
3751 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(
N, OpNo);
break;
3752 case ISD::ATOMIC_STORE:
3753 Res = SoftPromoteHalfOp_ATOMIC_STORE(
N, OpNo);
3756 Res = SoftPromoteHalfOp_STACKMAP(
N, OpNo);
3758 case ISD::PATCHPOINT:
3759 Res = SoftPromoteHalfOp_PATCHPOINT(
N, OpNo);
3769 "Invalid operand expansion");
3771 ReplaceValueWith(
SDValue(
N, 0), Res);
3776 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3778 return DAG.getNode(ISD::BITCAST, SDLoc(
N),
N->getValueType(0), Op0);
3781SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(
SDNode *
N,
unsigned OpNo) {
3782 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
3783 SDValue Op = GetSoftPromotedHalf(
N->getOperand(OpNo));
3784 return DAG.getNode(
N->getOpcode(), SDLoc(
N), MVT::Other,
N->getOperand(0),
3790 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
3795 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.
getValueType());
3797 Op1 = GetSoftPromotedHalf(Op1);
3800 return DAG.getNode(
N->getOpcode(), dl,
N->getValueType(0),
N->getOperand(0),
3804SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(
SDNode *
N) {
3805 EVT RVT =
N->getValueType(0);
3806 bool IsStrict =
N->isStrictFPOpcode();
3807 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3808 EVT SVT =
Op.getValueType();
3809 Op = GetSoftPromotedHalf(
N->getOperand(IsStrict ? 1 : 0));
3813 {RVT, MVT::Other}, {
N->getOperand(0),
Op});
3815 ReplaceValueWith(
SDValue(
N, 0), Res);
3822SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(
SDNode *
N) {
3823 EVT RVT =
N->getValueType(0);
3824 bool IsStrict =
N->isStrictFPOpcode();
3825 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3826 EVT SVT =
Op.getValueType();
3829 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3830 Op = GetSoftPromotedHalf(
Op);
3834 {
N->getOperand(0),
Op});
3835 Op = DAG.getNode(
N->getOpcode(), dl, {RVT, MVT::Other},
3836 {Op.getValue(1), Op});
3837 ReplaceValueWith(
SDValue(
N, 1),
Op.getValue(1));
3843 return DAG.getNode(
N->getOpcode(), dl, RVT, Res);
3846SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(
SDNode *
N) {
3847 EVT RVT =
N->getValueType(0);
3849 EVT SVT =
Op.getValueType();
3852 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
Op.getValueType());
3854 Op = GetSoftPromotedHalf(
Op);
3858 return DAG.getNode(
N->getOpcode(), dl,
N->getValueType(0), Res,
3864 assert(OpNo == 0 &&
"Can only soften the comparison values");
3870 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3872 Op0 = GetSoftPromotedHalf(Op0);
3873 Op1 = GetSoftPromotedHalf(Op1);
3877 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3878 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3881 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
3891 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.
getValueType());
3893 Op0 = GetSoftPromotedHalf(Op0);
3894 Op1 = GetSoftPromotedHalf(Op1);
3898 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3899 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3901 return DAG.getSetCC(SDLoc(
N),
N->getValueType(0), Op0, Op1, CCCode);
3904SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(
SDNode *
N,
unsigned OpNo) {
3905 assert(OpNo == 1 &&
"Can only soften the stored value!");
3910 assert(!
ST->isTruncatingStore() &&
"Unexpected truncating store.");
3911 SDValue Promoted = GetSoftPromotedHalf(Val);
3912 return DAG.getStore(
ST->getChain(), dl, Promoted,
ST->getBasePtr(),
3913 ST->getMemOperand());
3916SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(
SDNode *
N,
3918 assert(OpNo == 1 &&
"Can only soften the stored value!");
3923 SDValue Promoted = GetSoftPromotedHalf(Val);
3924 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.
getValueType(),
3925 ST->getChain(), Promoted,
ST->getBasePtr(),
3926 ST->getMemOperand());
3929SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
3933 NewOps[OpNo] = GetSoftPromotedHalf(
Op);
3935 DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getVTList(), NewOps);
3937 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
3948 NewOps[OpNo] = GetSoftPromotedHalf(
Op);
3950 DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getVTList(), NewOps);
3952 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static bool isSigned(unsigned int Opcode)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
APInt bitcastToAPInt() const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
void push_back(const T &Elt)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SHL
Shift and rotation operations.
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
LLVM_ABI Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOSPI(EVT RetVT)
getSINCOSPI - Return the SINCOSPI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getMODF(EVT RetVT)
getMODF - Return the MODF_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getCOS(EVT RetVT)
Return the COS_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSIN(EVT RetVT)
Return the SIN_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getPOW(EVT RetVT)
getPOW - Return the POW_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOS(EVT RetVT)
getSINCOS - Return the SINCOS_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
static LLVM_ABI const fltSemantics & PPCDoubleDouble() LLVM_READNONE
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)