LLVM 20.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::STRICT_FACOS:
80 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
81 case ISD::STRICT_FASIN:
82 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
83 case ISD::STRICT_FATAN:
84 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
85 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
86 case ISD::STRICT_FCEIL:
87 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
88 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
89 case ISD::STRICT_FCOS:
90 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
91 case ISD::STRICT_FCOSH:
92 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
93 case ISD::STRICT_FDIV:
94 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
95 case ISD::STRICT_FEXP:
96 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
97 case ISD::STRICT_FEXP2:
98 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
99 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
100 case ISD::STRICT_FFLOOR:
101 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
102 case ISD::STRICT_FLOG:
103 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
104 case ISD::STRICT_FLOG2:
105 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
106 case ISD::STRICT_FLOG10:
107 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
108 case ISD::STRICT_FMA:
109 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
110 case ISD::STRICT_FMUL:
111 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
112 case ISD::STRICT_FNEARBYINT:
113 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
114 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
115 case ISD::STRICT_FP_EXTEND:
116 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
117 case ISD::STRICT_FP_ROUND:
118 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
119 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
120 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
121 case ISD::STRICT_FPOW:
122 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
123 case ISD::STRICT_FPOWI:
124 case ISD::FPOWI:
125 case ISD::FLDEXP:
126 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
127 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
128 case ISD::STRICT_FREM:
129 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
130 case ISD::STRICT_FRINT:
131 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
132 case ISD::STRICT_FROUND:
133 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
134 case ISD::STRICT_FROUNDEVEN:
135 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
136 case ISD::STRICT_FSIN:
137 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
138 case ISD::STRICT_FSINH:
139 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
140 case ISD::STRICT_FSQRT:
141 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
142 case ISD::STRICT_FSUB:
143 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
144 case ISD::STRICT_FTAN:
145 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
146 case ISD::STRICT_FTANH:
147 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
148 case ISD::STRICT_FTRUNC:
149 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
150 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
151 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
152 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
153 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
154 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
155 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
156 case ISD::STRICT_SINT_TO_FP:
157 case ISD::STRICT_UINT_TO_FP:
158 case ISD::SINT_TO_FP:
159 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
160 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
161 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
162 case ISD::VECREDUCE_FADD:
163 case ISD::VECREDUCE_FMUL:
164 case ISD::VECREDUCE_FMIN:
165 case ISD::VECREDUCE_FMAX:
166 case ISD::VECREDUCE_FMAXIMUM:
167 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
168 case ISD::VECREDUCE_SEQ_FADD:
169 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
170 // clang-format on
171 }
172
173 // If R is null, the sub-method took care of registering the result.
174 if (R.getNode()) {
175 assert(R.getNode() != N);
176 SetSoftenedFloat(SDValue(N, ResNo), R);
177 }
178}
179
180SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
181 bool IsStrict = N->isStrictFPOpcode();
182 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
183 unsigned Offset = IsStrict ? 1 : 0;
184 assert(N->getNumOperands() == (1 + Offset) &&
185 "Unexpected number of operands!");
186 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
187 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
188 TargetLowering::MakeLibCallOptions CallOptions;
189 EVT OpVT = N->getOperand(0 + Offset).getValueType();
190 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
191 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
192 CallOptions, SDLoc(N),
193 Chain);
194 if (IsStrict)
195 ReplaceValueWith(SDValue(N, 1), Tmp.second);
196 return Tmp.first;
197}
198
199SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
200 bool IsStrict = N->isStrictFPOpcode();
201 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
202 unsigned Offset = IsStrict ? 1 : 0;
203 assert(N->getNumOperands() == (2 + Offset) &&
204 "Unexpected number of operands!");
205 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
206 GetSoftenedFloat(N->getOperand(1 + Offset)) };
207 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
208 TargetLowering::MakeLibCallOptions CallOptions;
209 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
210 N->getOperand(1 + Offset).getValueType() };
211 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
212 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
213 CallOptions, SDLoc(N),
214 Chain);
215 if (IsStrict)
216 ReplaceValueWith(SDValue(N, 1), Tmp.second);
217 return Tmp.first;
218}
219
220SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
221 return BitConvertToInteger(N->getOperand(0));
222}
223
224SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
225 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
226 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
227 GetSoftenedFloat(N->getOperand(0)));
228}
229
230SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
231 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
232 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
233 GetSoftenedFloat(N->getOperand(0)));
234 return NewFence;
235}
236
237SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
238 unsigned ResNo) {
239 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
240 return BitConvertToInteger(Op);
241}
242
243SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
244 // Convert the inputs to integers, and build a new pair out of them.
245 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
246 TLI.getTypeToTransformTo(*DAG.getContext(),
247 N->getValueType(0)),
248 BitConvertToInteger(N->getOperand(0)),
249 BitConvertToInteger(N->getOperand(1)));
250}
251
252SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
253 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
254 // In ppcf128, the high 64 bits are always first in memory regardless
255 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
256 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
257 // way. However, APInt's are serialized in an Endian-sensitive fashion,
258 // so on big-Endian targets, the two doubles are output in the wrong
259 // order. Fix this by manually flipping the order of the high 64 bits
260 // and the low 64 bits here.
261 if (DAG.getDataLayout().isBigEndian() &&
262 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
263 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
264 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
265 APInt Val(128, words);
266 return DAG.getConstant(Val, SDLoc(CN),
267 TLI.getTypeToTransformTo(*DAG.getContext(),
268 CN->getValueType(0)));
269 } else {
270 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
271 TLI.getTypeToTransformTo(*DAG.getContext(),
272 CN->getValueType(0)));
273 }
274}
275
276SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
277 SDValue Src = N->getOperand(0);
278 assert(Src.getValueType() == MVT::ppcf128 &&
279 "In floats only ppcf128 can be extracted by element!");
280 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
281 N->getValueType(0).changeTypeToInteger(),
282 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
283}
284
285SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
286 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
287 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
288 NewOp.getValueType().getVectorElementType(),
289 NewOp, N->getOperand(1));
290}
291
292SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
293 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
294 unsigned Size = NVT.getSizeInBits();
295
296 // Mask = ~(1 << (Size-1))
297 APInt API = APInt::getAllOnes(Size);
298 API.clearBit(Size - 1);
299 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
300 SDValue Op = GetSoftenedFloat(N->getOperand(0));
301 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
302}
303
304SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
305 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
306 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
308 RTLIB::FMIN_F32,
309 RTLIB::FMIN_F64,
310 RTLIB::FMIN_F80,
311 RTLIB::FMIN_F128,
312 RTLIB::FMIN_PPCF128));
313}
314
315SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
316 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
317 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
318 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
319 RTLIB::FMAX_F32,
320 RTLIB::FMAX_F64,
321 RTLIB::FMAX_F80,
322 RTLIB::FMAX_F128,
323 RTLIB::FMAX_PPCF128));
324}
325
326SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
327 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
328 RTLIB::ADD_F32,
329 RTLIB::ADD_F64,
330 RTLIB::ADD_F80,
331 RTLIB::ADD_F128,
332 RTLIB::ADD_PPCF128));
333}
334
335SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
336 return SoftenFloatRes_Unary(
337 N, GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64,
338 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128));
339}
340
341SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
342 return SoftenFloatRes_Unary(
343 N, GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64,
344 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128));
345}
346
347SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
348 return SoftenFloatRes_Unary(
349 N, GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64,
350 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128));
351}
352
353SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
354 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
355 RTLIB::CBRT_F32,
356 RTLIB::CBRT_F64,
357 RTLIB::CBRT_F80,
358 RTLIB::CBRT_F128,
359 RTLIB::CBRT_PPCF128));
360}
361
362SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
363 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
364 RTLIB::CEIL_F32,
365 RTLIB::CEIL_F64,
366 RTLIB::CEIL_F80,
367 RTLIB::CEIL_F128,
368 RTLIB::CEIL_PPCF128));
369}
370
371SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
372 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
373 SDValue RHS = BitConvertToInteger(N->getOperand(1));
374 SDLoc dl(N);
375
376 EVT LVT = LHS.getValueType();
377 EVT RVT = RHS.getValueType();
378
379 unsigned LSize = LVT.getSizeInBits();
380 unsigned RSize = RVT.getSizeInBits();
381
382 // First get the sign bit of second operand.
383 SDValue SignBit = DAG.getNode(
384 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
385 DAG.getConstant(RSize - 1, dl,
386 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
387 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
388
389 // Shift right or sign-extend it if the two operands have different types.
390 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
391 if (SizeDiff > 0) {
392 SignBit =
393 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
394 DAG.getConstant(SizeDiff, dl,
395 TLI.getShiftAmountTy(SignBit.getValueType(),
396 DAG.getDataLayout())));
397 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
398 } else if (SizeDiff < 0) {
399 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
400 SignBit =
401 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
402 DAG.getConstant(-SizeDiff, dl,
403 TLI.getShiftAmountTy(SignBit.getValueType(),
404 DAG.getDataLayout())));
405 }
406
407 // Clear the sign bit of the first operand.
408 SDValue Mask = DAG.getNode(
409 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
410 DAG.getConstant(LSize - 1, dl,
411 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
412 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
413 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
414
415 // Or the value with the sign bit.
416 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
417}
418
419SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
420 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
421 RTLIB::COS_F32,
422 RTLIB::COS_F64,
423 RTLIB::COS_F80,
424 RTLIB::COS_F128,
425 RTLIB::COS_PPCF128));
426}
427
428SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
429 return SoftenFloatRes_Unary(
430 N, GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64,
431 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128));
432}
433
434SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
435 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
436 RTLIB::DIV_F32,
437 RTLIB::DIV_F64,
438 RTLIB::DIV_F80,
439 RTLIB::DIV_F128,
440 RTLIB::DIV_PPCF128));
441}
442
443SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
444 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
445 RTLIB::EXP_F32,
446 RTLIB::EXP_F64,
447 RTLIB::EXP_F80,
448 RTLIB::EXP_F128,
449 RTLIB::EXP_PPCF128));
450}
451
452SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
453 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
454 RTLIB::EXP2_F32,
455 RTLIB::EXP2_F64,
456 RTLIB::EXP2_F80,
457 RTLIB::EXP2_F128,
458 RTLIB::EXP2_PPCF128));
459}
460
461SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
462 return SoftenFloatRes_Unary(
463 N,
464 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
465 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
466}
467
468SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
469 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
470 RTLIB::FLOOR_F32,
471 RTLIB::FLOOR_F64,
472 RTLIB::FLOOR_F80,
473 RTLIB::FLOOR_F128,
474 RTLIB::FLOOR_PPCF128));
475}
476
477SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
478 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
479 RTLIB::LOG_F32,
480 RTLIB::LOG_F64,
481 RTLIB::LOG_F80,
482 RTLIB::LOG_F128,
483 RTLIB::LOG_PPCF128));
484}
485
486SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
487 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
488 RTLIB::LOG2_F32,
489 RTLIB::LOG2_F64,
490 RTLIB::LOG2_F80,
491 RTLIB::LOG2_F128,
492 RTLIB::LOG2_PPCF128));
493}
494
495SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
497 RTLIB::LOG10_F32,
498 RTLIB::LOG10_F64,
499 RTLIB::LOG10_F80,
500 RTLIB::LOG10_F128,
501 RTLIB::LOG10_PPCF128));
502}
503
504SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
505 bool IsStrict = N->isStrictFPOpcode();
506 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
507 unsigned Offset = IsStrict ? 1 : 0;
508 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
509 GetSoftenedFloat(N->getOperand(1 + Offset)),
510 GetSoftenedFloat(N->getOperand(2 + Offset)) };
511 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
512 TargetLowering::MakeLibCallOptions CallOptions;
513 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
514 N->getOperand(1 + Offset).getValueType(),
515 N->getOperand(2 + Offset).getValueType() };
516 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
517 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
518 GetFPLibCall(N->getValueType(0),
519 RTLIB::FMA_F32,
520 RTLIB::FMA_F64,
521 RTLIB::FMA_F80,
522 RTLIB::FMA_F128,
523 RTLIB::FMA_PPCF128),
524 NVT, Ops, CallOptions, SDLoc(N), Chain);
525 if (IsStrict)
526 ReplaceValueWith(SDValue(N, 1), Tmp.second);
527 return Tmp.first;
528}
529
530SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
531 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
532 RTLIB::MUL_F32,
533 RTLIB::MUL_F64,
534 RTLIB::MUL_F80,
535 RTLIB::MUL_F128,
536 RTLIB::MUL_PPCF128));
537}
538
539SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
540 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
541 RTLIB::NEARBYINT_F32,
542 RTLIB::NEARBYINT_F64,
543 RTLIB::NEARBYINT_F80,
544 RTLIB::NEARBYINT_F128,
545 RTLIB::NEARBYINT_PPCF128));
546}
547
548SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
549 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
550 SDLoc dl(N);
551
552 // Expand Y = FNEG(X) -> Y = X ^ sign mask
553 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
554 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
555 DAG.getConstant(SignMask, dl, NVT));
556}
557
558SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
559 bool IsStrict = N->isStrictFPOpcode();
560 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
561 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
562
563 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
564
565 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
566 Op = GetPromotedFloat(Op);
567 // If the promotion did the FP_EXTEND to the destination type for us,
568 // there's nothing left to do here.
569 if (Op.getValueType() == N->getValueType(0)) {
570 if (IsStrict)
571 ReplaceValueWith(SDValue(N, 1), Chain);
572 return BitConvertToInteger(Op);
573 }
574 }
575
576 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
577 // -> f32, so proceed in two stages. Also, it's entirely possible for both
578 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
579 // than FP16_TO_FP.
580 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
581 N->getValueType(0) != MVT::f32) {
582 if (IsStrict) {
583 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
584 { MVT::f32, MVT::Other }, { Chain, Op });
585 Chain = Op.getValue(1);
586 } else {
587 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
588 }
589 }
590
591 if (Op.getValueType() == MVT::bf16) {
592 // FIXME: Need ReplaceValueWith on chain in strict case
593 return SoftenFloatRes_BF16_TO_FP(N);
594 }
595
596 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
597 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
598 TargetLowering::MakeLibCallOptions CallOptions;
599 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
600 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
601 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
602 CallOptions, SDLoc(N),
603 Chain);
604 if (IsStrict)
605 ReplaceValueWith(SDValue(N, 1), Tmp.second);
606 return Tmp.first;
607}
608
609// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
610// nodes?
611SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
612 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
613 SDValue Op = N->getOperand(0);
614 TargetLowering::MakeLibCallOptions CallOptions;
615 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
616 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
617 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
618 CallOptions, SDLoc(N)).first;
619 if (N->getValueType(0) == MVT::f32)
620 return Res32;
621
622 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
623 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
624 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
625 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
626}
627
628// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
629// nodes?
630SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
631 assert(N->getValueType(0) == MVT::f32 &&
632 "Can only soften BF16_TO_FP with f32 result");
633 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
634 SDValue Op = N->getOperand(0);
635 SDLoc DL(N);
636 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
637 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
638 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
639 DAG.getShiftAmountConstant(16, NVT, DL));
640 return Res;
641}
642
643SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
644 bool IsStrict = N->isStrictFPOpcode();
645 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
646 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
647 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
648 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
650 TargetLowering::MakeLibCallOptions CallOptions;
651 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
652 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
653 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
654 CallOptions, SDLoc(N),
655 Chain);
656 if (IsStrict)
657 ReplaceValueWith(SDValue(N, 1), Tmp.second);
658 return Tmp.first;
659}
660
661SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
662 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
663 RTLIB::POW_F32,
664 RTLIB::POW_F64,
665 RTLIB::POW_F80,
666 RTLIB::POW_F128,
667 RTLIB::POW_PPCF128));
668}
669
670SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
671 bool IsStrict = N->isStrictFPOpcode();
672 unsigned Offset = IsStrict ? 1 : 0;
673 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
674 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
675 "Unsupported power type!");
676 bool IsPowI =
677 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
678
679 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
680 : RTLIB::getLDEXP(N->getValueType(0));
681 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
682 if (!TLI.getLibcallName(LC)) {
683 // Some targets don't have a powi libcall; use pow instead.
684 // FIXME: Implement this if some target needs it.
685 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
686 return DAG.getUNDEF(N->getValueType(0));
687 }
688
689 if (DAG.getLibInfo().getIntSize() !=
690 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
691 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
692 // would use the wrong type for the argument.
693 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
694 return DAG.getUNDEF(N->getValueType(0));
695 }
696
697 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
698 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
699 N->getOperand(1 + Offset) };
700 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
701 TargetLowering::MakeLibCallOptions CallOptions;
702 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
703 N->getOperand(1 + Offset).getValueType() };
704 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
705 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
706 CallOptions, SDLoc(N),
707 Chain);
708 if (IsStrict)
709 ReplaceValueWith(SDValue(N, 1), Tmp.second);
710 return Tmp.first;
711}
712
713SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
714 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
715 EVT VT0 = N->getValueType(0);
716 EVT VT1 = N->getValueType(1);
717 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
718
719 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
720 // If the exponent does not match with sizeof(int) a libcall would use the
721 // wrong type for the argument.
722 // TODO: Should be able to handle mismatches.
723 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
724 return DAG.getUNDEF(N->getValueType(0));
725 }
726
727 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
728 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
729
730 SDLoc DL(N);
731
732 TargetLowering::MakeLibCallOptions CallOptions;
733 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
734 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
735
736 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
737 // but we only really need to handle the 0th one for softening anyway.
738 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
739
740 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
741 /*Chain=*/SDValue());
742 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
743 auto PtrInfo =
744 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
745
746 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
747
748 ReplaceValueWith(SDValue(N, 1), LoadExp);
749 return ReturnVal;
750}
751
752SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
753 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
754 RTLIB::REM_F32,
755 RTLIB::REM_F64,
756 RTLIB::REM_F80,
757 RTLIB::REM_F128,
758 RTLIB::REM_PPCF128));
759}
760
761SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
762 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
763 RTLIB::RINT_F32,
764 RTLIB::RINT_F64,
765 RTLIB::RINT_F80,
766 RTLIB::RINT_F128,
767 RTLIB::RINT_PPCF128));
768}
769
770SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
771 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
772 RTLIB::ROUND_F32,
773 RTLIB::ROUND_F64,
774 RTLIB::ROUND_F80,
775 RTLIB::ROUND_F128,
776 RTLIB::ROUND_PPCF128));
777}
778
779SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
780 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
781 RTLIB::ROUNDEVEN_F32,
782 RTLIB::ROUNDEVEN_F64,
783 RTLIB::ROUNDEVEN_F80,
784 RTLIB::ROUNDEVEN_F128,
785 RTLIB::ROUNDEVEN_PPCF128));
786}
787
788SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
789 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
790 RTLIB::SIN_F32,
791 RTLIB::SIN_F64,
792 RTLIB::SIN_F80,
793 RTLIB::SIN_F128,
794 RTLIB::SIN_PPCF128));
795}
796
797SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
798 return SoftenFloatRes_Unary(
799 N, GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64,
800 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128));
801}
802
803SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
804 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
805 RTLIB::SQRT_F32,
806 RTLIB::SQRT_F64,
807 RTLIB::SQRT_F80,
808 RTLIB::SQRT_F128,
809 RTLIB::SQRT_PPCF128));
810}
811
812SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
813 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
814 RTLIB::SUB_F32,
815 RTLIB::SUB_F64,
816 RTLIB::SUB_F80,
817 RTLIB::SUB_F128,
818 RTLIB::SUB_PPCF128));
819}
820
821SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
822 return SoftenFloatRes_Unary(
823 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
824 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
825}
826
827SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
828 return SoftenFloatRes_Unary(
829 N, GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64,
830 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128));
831}
832
833SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
834 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
835 RTLIB::TRUNC_F32,
836 RTLIB::TRUNC_F64,
837 RTLIB::TRUNC_F80,
838 RTLIB::TRUNC_F128,
839 RTLIB::TRUNC_PPCF128));
840}
841
842SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
843 LoadSDNode *L = cast<LoadSDNode>(N);
844 EVT VT = N->getValueType(0);
845 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
846 SDLoc dl(N);
847
848 auto MMOFlags =
849 L->getMemOperand()->getFlags() &
850 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
851 SDValue NewL;
852 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
853 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
854 L->getChain(), L->getBasePtr(), L->getOffset(),
855 L->getPointerInfo(), NVT, L->getOriginalAlign(),
856 MMOFlags, L->getAAInfo());
857 // Legalized the chain result - switch anything that used the old chain to
858 // use the new one.
859 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
860 return NewL;
861 }
862
863 // Do a non-extending load followed by FP_EXTEND.
864 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
865 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
866 L->getPointerInfo(), L->getMemoryVT(),
867 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
868 // Legalized the chain result - switch anything that used the old chain to
869 // use the new one.
870 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
871 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
872 return BitConvertToInteger(ExtendNode);
873}
874
875SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
876 AtomicSDNode *L = cast<AtomicSDNode>(N);
877 EVT VT = N->getValueType(0);
878 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
879 SDLoc dl(N);
880
881 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
882 SDValue NewL =
883 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
884 {L->getChain(), L->getBasePtr()}, L->getMemOperand());
885
886 // Legalized the chain result - switch anything that used the old chain to
887 // use the new one.
888 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
889 return NewL;
890 }
891
892 report_fatal_error("softening fp extending atomic load not handled");
893}
894
895SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
896 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
897 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
898 return DAG.getSelect(SDLoc(N),
899 LHS.getValueType(), N->getOperand(0), LHS, RHS);
900}
901
902SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
903 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
904 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
905 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
906 LHS.getValueType(), N->getOperand(0),
907 N->getOperand(1), LHS, RHS, N->getOperand(4));
908}
909
910SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
911 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
912 N->getValueType(0)));
913}
914
915SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
916 SDValue Chain = N->getOperand(0); // Get the chain.
917 SDValue Ptr = N->getOperand(1); // Get the pointer.
918 EVT VT = N->getValueType(0);
919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
920 SDLoc dl(N);
921
922 SDValue NewVAARG;
923 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
924 N->getConstantOperandVal(3));
925
926 // Legalized the chain result - switch anything that used the old chain to
927 // use the new one.
928 if (N != NewVAARG.getValue(1).getNode())
929 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
930 return NewVAARG;
931}
932
933SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
934 bool IsStrict = N->isStrictFPOpcode();
935 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
936 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
937 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
938 EVT RVT = N->getValueType(0);
939 EVT NVT = EVT();
940 SDLoc dl(N);
941
942 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
943 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
944 // match. Look for an appropriate libcall.
945 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
946 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
947 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
948 NVT = (MVT::SimpleValueType)t;
949 // The source needs to big enough to hold the operand.
950 if (NVT.bitsGE(SVT))
951 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
952 }
953 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
954
955 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
956 // Sign/zero extend the argument if the libcall takes a larger type.
957 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
958 NVT, N->getOperand(IsStrict ? 1 : 0));
959 TargetLowering::MakeLibCallOptions CallOptions;
960 CallOptions.setSExt(Signed);
961 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
962 std::pair<SDValue, SDValue> Tmp =
963 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
964 Op, CallOptions, dl, Chain);
965
966 if (IsStrict)
967 ReplaceValueWith(SDValue(N, 1), Tmp.second);
968 return Tmp.first;
969}
970
971SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
972 // Expand and soften recursively.
973 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
974 return SDValue();
975}
976
977SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
978 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
979 return SDValue();
980}
981
982//===----------------------------------------------------------------------===//
983// Convert Float Operand to Integer
984//===----------------------------------------------------------------------===//
985
986bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
987 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
988 SDValue Res = SDValue();
989
990 switch (N->getOpcode()) {
991 default:
992#ifndef NDEBUG
993 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
994 N->dump(&DAG); dbgs() << "\n";
995#endif
996 report_fatal_error("Do not know how to soften this operator's operand!");
997
998 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
999 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1000 case ISD::STRICT_FP_TO_FP16:
1001 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1002 case ISD::FP_TO_BF16:
1003 case ISD::STRICT_FP_TO_BF16:
1004 case ISD::STRICT_FP_ROUND:
1005 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1006 case ISD::STRICT_FP_TO_SINT:
1007 case ISD::STRICT_FP_TO_UINT:
1008 case ISD::FP_TO_SINT:
1009 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1010 case ISD::FP_TO_SINT_SAT:
1011 case ISD::FP_TO_UINT_SAT:
1012 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1013 case ISD::STRICT_LROUND:
1014 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1015 case ISD::STRICT_LLROUND:
1016 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1017 case ISD::STRICT_LRINT:
1018 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1019 case ISD::STRICT_LLRINT:
1020 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1021 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1022 case ISD::STRICT_FSETCC:
1023 case ISD::STRICT_FSETCCS:
1024 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1025 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1026 case ISD::ATOMIC_STORE:
1027 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1028 break;
1029 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1030 }
1031
1032 // If the result is null, the sub-method took care of registering results etc.
1033 if (!Res.getNode()) return false;
1034
1035 // If the result is N, the sub-method updated N in place. Tell the legalizer
1036 // core about this to re-analyze.
1037 if (Res.getNode() == N)
1038 return true;
1039
1040 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1041 "Invalid operand softening");
1042
1043 ReplaceValueWith(SDValue(N, 0), Res);
1044 return false;
1045}
1046
1047SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1048 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
1049
1050 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
1051}
1052
1053SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1054 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1055 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1056 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1057 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1058 N->getOpcode() == ISD::FP_TO_BF16 ||
1059 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1060 N->getOpcode() == ISD::STRICT_FP_ROUND);
1061
1062 bool IsStrict = N->isStrictFPOpcode();
1063 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1064 EVT SVT = Op.getValueType();
1065 EVT RVT = N->getValueType(0);
1066 EVT FloatRVT = RVT;
1067 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1068 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1069 FloatRVT = MVT::f16;
1070 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1071 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1072 FloatRVT = MVT::bf16;
1073
1074 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
1075 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1076
1077 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1078 Op = GetSoftenedFloat(Op);
1079 TargetLowering::MakeLibCallOptions CallOptions;
1080 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1081 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1082 CallOptions, SDLoc(N),
1083 Chain);
1084 if (IsStrict) {
1085 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1086 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1087 return SDValue();
1088 }
1089 return Tmp.first;
1090}
1091
1092SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1093 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1094 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1095
1096 EVT VT = NewLHS.getValueType();
1097 NewLHS = GetSoftenedFloat(NewLHS);
1098 NewRHS = GetSoftenedFloat(NewRHS);
1099 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1100 N->getOperand(2), N->getOperand(3));
1101
1102 // If softenSetCCOperands returned a scalar, we need to compare the result
1103 // against zero to select between true and false values.
1104 if (!NewRHS.getNode()) {
1105 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1106 CCCode = ISD::SETNE;
1107 }
1108
1109 // Update N to have the operands specified.
1110 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1111 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1112 N->getOperand(4)),
1113 0);
1114}
1115
1116// Even if the result type is legal, no libcall may exactly match. (e.g. We
1117// don't have FP-i8 conversions) This helper method looks for an appropriate
1118// promoted libcall.
1119static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1120 bool Signed) {
1121 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1122 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1123 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1124 ++IntVT) {
1125 Promoted = (MVT::SimpleValueType)IntVT;
1126 // The type needs to big enough to hold the result.
1127 if (Promoted.bitsGE(RetVT))
1128 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1129 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1130 }
1131 return LC;
1132}
1133
1134SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1135 bool IsStrict = N->isStrictFPOpcode();
1136 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1137 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1138
1139 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1140 EVT SVT = Op.getValueType();
1141 EVT RVT = N->getValueType(0);
1142 EVT NVT = EVT();
1143 SDLoc dl(N);
1144
1145 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1146 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1147 // match, eg. we don't have fp -> i8 conversions.
1148 // Look for an appropriate libcall.
1149 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1150 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1151 "Unsupported FP_TO_XINT!");
1152
1153 Op = GetSoftenedFloat(Op);
1154 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1155 TargetLowering::MakeLibCallOptions CallOptions;
1156 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1157 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1158 CallOptions, dl, Chain);
1159
1160 // Truncate the result if the libcall returns a larger type.
1161 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1162
1163 if (!IsStrict)
1164 return Res;
1165
1166 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1167 ReplaceValueWith(SDValue(N, 0), Res);
1168 return SDValue();
1169}
1170
1171SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1172 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1173 return Res;
1174}
1175
1176SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1177 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1178 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1179
1180 EVT VT = NewLHS.getValueType();
1181 NewLHS = GetSoftenedFloat(NewLHS);
1182 NewRHS = GetSoftenedFloat(NewRHS);
1183 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1184 N->getOperand(0), N->getOperand(1));
1185
1186 // If softenSetCCOperands returned a scalar, we need to compare the result
1187 // against zero to select between true and false values.
1188 if (!NewRHS.getNode()) {
1189 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1190 CCCode = ISD::SETNE;
1191 }
1192
1193 // Update N to have the operands specified.
1194 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1195 N->getOperand(2), N->getOperand(3),
1196 DAG.getCondCode(CCCode)),
1197 0);
1198}
1199
1200SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1201 bool IsStrict = N->isStrictFPOpcode();
1202 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1203 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1204 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1205 ISD::CondCode CCCode =
1206 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1207
1208 EVT VT = Op0.getValueType();
1209 SDValue NewLHS = GetSoftenedFloat(Op0);
1210 SDValue NewRHS = GetSoftenedFloat(Op1);
1211 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1212 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1213
1214 // Update N to have the operands specified.
1215 if (NewRHS.getNode()) {
1216 if (IsStrict)
1217 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1218 NewRHS, DAG.getCondCode(CCCode));
1219 else
1220 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1221 DAG.getCondCode(CCCode)), 0);
1222 }
1223
1224 // Otherwise, softenSetCCOperands returned a scalar, use it.
1225 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1226 "Unexpected setcc expansion!");
1227
1228 if (IsStrict) {
1229 ReplaceValueWith(SDValue(N, 0), NewLHS);
1230 ReplaceValueWith(SDValue(N, 1), Chain);
1231 return SDValue();
1232 }
1233 return NewLHS;
1234}
1235
1236SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1237 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1238 assert(OpNo == 1 && "Can only soften the stored value!");
1239 StoreSDNode *ST = cast<StoreSDNode>(N);
1240 SDValue Val = ST->getValue();
1241 SDLoc dl(N);
1242
1243 if (ST->isTruncatingStore())
1244 // Do an FP_ROUND followed by a non-truncating store.
1245 Val = BitConvertToInteger(
1246 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1247 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1248 else
1249 Val = GetSoftenedFloat(Val);
1250
1251 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1252 ST->getMemOperand());
1253}
1254
1255SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1256 assert(OpNo == 1 && "Can only soften the stored value!");
1257 AtomicSDNode *ST = cast<AtomicSDNode>(N);
1258 SDValue Val = ST->getVal();
1259 EVT VT = Val.getValueType();
1260 SDLoc dl(N);
1261
1262 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1263
1264 SDValue NewVal = GetSoftenedFloat(Val);
1265 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal,
1266 ST->getBasePtr(), ST->getMemOperand());
1267}
1268
1269SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1270 SDValue LHS = N->getOperand(0);
1271 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1272 SDLoc dl(N);
1273
1274 EVT LVT = LHS.getValueType();
1275 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1276 EVT RVT = RHS.getValueType();
1277
1278 unsigned LSize = LVT.getSizeInBits();
1279 unsigned RSize = RVT.getSizeInBits();
1280
1281 // Shift right or sign-extend it if the two operands have different types.
1282 int SizeDiff = RSize - LSize;
1283 if (SizeDiff > 0) {
1284 RHS =
1285 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1286 DAG.getConstant(SizeDiff, dl,
1287 TLI.getShiftAmountTy(RHS.getValueType(),
1288 DAG.getDataLayout())));
1289 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1290 } else if (SizeDiff < 0) {
1291 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1292 RHS =
1293 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1294 DAG.getConstant(-SizeDiff, dl,
1295 TLI.getShiftAmountTy(RHS.getValueType(),
1296 DAG.getDataLayout())));
1297 }
1298
1299 RHS = DAG.getBitcast(LVT, RHS);
1300 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1301}
1302
1303SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1304 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1305 bool IsStrict = N->isStrictFPOpcode();
1306 unsigned Offset = IsStrict ? 1 : 0;
1307 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1308 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1309 TargetLowering::MakeLibCallOptions CallOptions;
1310 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1311 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1312 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1313 CallOptions, SDLoc(N),
1314 Chain);
1315 if (IsStrict) {
1316 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1317 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1318 return SDValue();
1319 }
1320
1321 return Tmp.first;
1322}
1323
1324SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1325 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1326 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1327 RTLIB::LROUND_F32,
1328 RTLIB::LROUND_F64,
1329 RTLIB::LROUND_F80,
1330 RTLIB::LROUND_F128,
1331 RTLIB::LROUND_PPCF128));
1332}
1333
1334SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1335 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1336 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1337 RTLIB::LLROUND_F32,
1338 RTLIB::LLROUND_F64,
1339 RTLIB::LLROUND_F80,
1340 RTLIB::LLROUND_F128,
1341 RTLIB::LLROUND_PPCF128));
1342}
1343
1344SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1345 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1346 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1347 RTLIB::LRINT_F32,
1348 RTLIB::LRINT_F64,
1349 RTLIB::LRINT_F80,
1350 RTLIB::LRINT_F128,
1351 RTLIB::LRINT_PPCF128));
1352}
1353
1354SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1355 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1356 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1357 RTLIB::LLRINT_F32,
1358 RTLIB::LLRINT_F64,
1359 RTLIB::LLRINT_F80,
1360 RTLIB::LLRINT_F128,
1361 RTLIB::LLRINT_PPCF128));
1362}
1363
1364//===----------------------------------------------------------------------===//
1365// Float Result Expansion
1366//===----------------------------------------------------------------------===//
1367
1368/// ExpandFloatResult - This method is called when the specified result of the
1369/// specified node is found to need expansion. At this point, the node may also
1370/// have invalid operands or may have other results that need promotion, we just
1371/// know that (at least) one result needs expansion.
1372void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1373 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1374 SDValue Lo, Hi;
1375 Lo = Hi = SDValue();
1376
1377 // See if the target wants to custom expand this node.
1378 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1379 return;
1380
1381 switch (N->getOpcode()) {
1382 default:
1383#ifndef NDEBUG
1384 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1385 N->dump(&DAG); dbgs() << "\n";
1386#endif
1387 report_fatal_error("Do not know how to expand the result of this "
1388 "operator!");
1389 // clang-format off
1390 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1391 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1392 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1393
1394 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1395 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1396 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1397 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1398 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1399 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1400
1401 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1402 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1403 case ISD::STRICT_FMINNUM:
1404 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1405 case ISD::STRICT_FMAXNUM:
1406 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1407 case ISD::STRICT_FADD:
1408 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1409 case ISD::STRICT_FACOS:
1410 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1411 case ISD::STRICT_FASIN:
1412 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1413 case ISD::STRICT_FATAN:
1414 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1415 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1416 case ISD::STRICT_FCEIL:
1417 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1418 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1419 case ISD::STRICT_FCOS:
1420 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1421 case ISD::STRICT_FCOSH:
1422 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1423 case ISD::STRICT_FDIV:
1424 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1425 case ISD::STRICT_FEXP:
1426 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1427 case ISD::STRICT_FEXP2:
1428 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1429 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1430 case ISD::STRICT_FFLOOR:
1431 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1432 case ISD::STRICT_FLOG:
1433 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1434 case ISD::STRICT_FLOG2:
1435 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1436 case ISD::STRICT_FLOG10:
1437 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1438 case ISD::STRICT_FMA:
1439 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1440 case ISD::STRICT_FMUL:
1441 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1442 case ISD::STRICT_FNEARBYINT:
1443 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1444 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1445 case ISD::STRICT_FP_EXTEND:
1446 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1447 case ISD::STRICT_FPOW:
1448 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1449 case ISD::STRICT_FPOWI:
1450 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1451 case ISD::FLDEXP:
1452 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1453 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1454 case ISD::STRICT_FRINT:
1455 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1456 case ISD::STRICT_FROUND:
1457 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1458 case ISD::STRICT_FROUNDEVEN:
1459 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1460 case ISD::STRICT_FSIN:
1461 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1462 case ISD::STRICT_FSINH:
1463 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1464 case ISD::STRICT_FSQRT:
1465 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1466 case ISD::STRICT_FSUB:
1467 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1468 case ISD::STRICT_FTAN:
1469 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1470 case ISD::STRICT_FTANH:
1471 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1472 case ISD::STRICT_FTRUNC:
1473 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1474 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1475 case ISD::STRICT_SINT_TO_FP:
1476 case ISD::STRICT_UINT_TO_FP:
1477 case ISD::SINT_TO_FP:
1478 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1479 case ISD::STRICT_FREM:
1480 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1481 // clang-format on
1482 }
1483
1484 // If Lo/Hi is null, the sub-method took care of registering results etc.
1485 if (Lo.getNode())
1486 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1487}
1488
1489void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1490 SDValue &Hi) {
1491 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1492 assert(NVT.getSizeInBits() == 64 &&
1493 "Do not know how to expand this float constant!");
1494 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1495 SDLoc dl(N);
1496 const fltSemantics &Sem = NVT.getFltSemantics();
1497 Lo = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 64)), dl, NVT);
1498 Hi = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 0)), dl, NVT);
1499}
1500
1501void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1502 SDValue &Lo, SDValue &Hi) {
1503 bool IsStrict = N->isStrictFPOpcode();
1504 unsigned Offset = IsStrict ? 1 : 0;
1505 SDValue Op = N->getOperand(0 + Offset);
1506 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1507 TargetLowering::MakeLibCallOptions CallOptions;
1508 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1509 Op, CallOptions, SDLoc(N),
1510 Chain);
1511 if (IsStrict)
1512 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1513 GetPairElements(Tmp.first, Lo, Hi);
1514}
1515
1516void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1517 SDValue &Lo, SDValue &Hi) {
1518 bool IsStrict = N->isStrictFPOpcode();
1519 unsigned Offset = IsStrict ? 1 : 0;
1520 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1521 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1522 TargetLowering::MakeLibCallOptions CallOptions;
1523 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1524 Ops, CallOptions, SDLoc(N),
1525 Chain);
1526 if (IsStrict)
1527 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1528 GetPairElements(Tmp.first, Lo, Hi);
1529}
1530
1531void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1532 SDValue &Hi) {
1533 assert(N->getValueType(0) == MVT::ppcf128 &&
1534 "Logic only correct for ppcf128!");
1535 SDLoc dl(N);
1536 SDValue Tmp;
1537 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1538 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1539 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1540 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1541 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1542 ISD::SETEQ);
1543}
1544
1545void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1546 SDValue &Hi) {
1547 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1548 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1549 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1550 RTLIB::FMIN_PPCF128), Lo, Hi);
1551}
1552
1553void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1554 SDValue &Hi) {
1555 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1556 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1557 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1558 RTLIB::FMAX_PPCF128), Lo, Hi);
1559}
1560
1561void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1562 SDValue &Hi) {
1563 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1564 RTLIB::ADD_F32, RTLIB::ADD_F64,
1565 RTLIB::ADD_F80, RTLIB::ADD_F128,
1566 RTLIB::ADD_PPCF128), Lo, Hi);
1567}
1568
1569void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1570 SDValue &Hi) {
1571 ExpandFloatRes_Unary(N,
1572 GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32,
1573 RTLIB::ACOS_F64, RTLIB::ACOS_F80,
1574 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128),
1575 Lo, Hi);
1576}
1577
1578void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1579 SDValue &Hi) {
1580 ExpandFloatRes_Unary(N,
1581 GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32,
1582 RTLIB::ASIN_F64, RTLIB::ASIN_F80,
1583 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128),
1584 Lo, Hi);
1585}
1586
1587void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1588 SDValue &Hi) {
1589 ExpandFloatRes_Unary(N,
1590 GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32,
1591 RTLIB::ATAN_F64, RTLIB::ATAN_F80,
1592 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128),
1593 Lo, Hi);
1594}
1595
1596void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1597 SDValue &Hi) {
1598 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1599 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1600 RTLIB::CBRT_F128,
1601 RTLIB::CBRT_PPCF128), Lo, Hi);
1602}
1603
1604void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1605 SDValue &Lo, SDValue &Hi) {
1606 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1607 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1608 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1609 RTLIB::CEIL_PPCF128), Lo, Hi);
1610}
1611
1612void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1613 SDValue &Lo, SDValue &Hi) {
1614 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1615 RTLIB::COPYSIGN_F32,
1616 RTLIB::COPYSIGN_F64,
1617 RTLIB::COPYSIGN_F80,
1618 RTLIB::COPYSIGN_F128,
1619 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1620}
1621
1622void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1623 SDValue &Lo, SDValue &Hi) {
1624 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1625 RTLIB::COS_F32, RTLIB::COS_F64,
1626 RTLIB::COS_F80, RTLIB::COS_F128,
1627 RTLIB::COS_PPCF128), Lo, Hi);
1628}
1629
1630void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1631 SDValue &Hi) {
1632 ExpandFloatRes_Unary(N,
1633 GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32,
1634 RTLIB::COSH_F64, RTLIB::COSH_F80,
1635 RTLIB::COSH_F128, RTLIB::COSH_PPCF128),
1636 Lo, Hi);
1637}
1638
1639void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1640 SDValue &Hi) {
1641 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1642 RTLIB::DIV_F32,
1643 RTLIB::DIV_F64,
1644 RTLIB::DIV_F80,
1645 RTLIB::DIV_F128,
1646 RTLIB::DIV_PPCF128), Lo, Hi);
1647}
1648
1649void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1650 SDValue &Lo, SDValue &Hi) {
1651 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1652 RTLIB::EXP_F32, RTLIB::EXP_F64,
1653 RTLIB::EXP_F80, RTLIB::EXP_F128,
1654 RTLIB::EXP_PPCF128), Lo, Hi);
1655}
1656
1657void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1658 SDValue &Lo, SDValue &Hi) {
1659 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1660 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1661 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1662 RTLIB::EXP2_PPCF128), Lo, Hi);
1663}
1664
1665void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1666 SDValue &Hi) {
1667 ExpandFloatRes_Unary(N,
1668 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1669 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1670 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1671 Lo, Hi);
1672}
1673
1674void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1675 SDValue &Lo, SDValue &Hi) {
1676 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1677 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1678 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1679 RTLIB::FLOOR_PPCF128), Lo, Hi);
1680}
1681
1682void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1683 SDValue &Lo, SDValue &Hi) {
1684 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1685 RTLIB::LOG_F32, RTLIB::LOG_F64,
1686 RTLIB::LOG_F80, RTLIB::LOG_F128,
1687 RTLIB::LOG_PPCF128), Lo, Hi);
1688}
1689
1690void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1691 SDValue &Lo, SDValue &Hi) {
1692 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1693 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1694 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1695 RTLIB::LOG2_PPCF128), Lo, Hi);
1696}
1697
1698void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1699 SDValue &Lo, SDValue &Hi) {
1700 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1701 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1702 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1703 RTLIB::LOG10_PPCF128), Lo, Hi);
1704}
1705
1706void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1707 SDValue &Hi) {
1708 bool IsStrict = N->isStrictFPOpcode();
1709 unsigned Offset = IsStrict ? 1 : 0;
1710 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1711 N->getOperand(2 + Offset) };
1712 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1713 TargetLowering::MakeLibCallOptions CallOptions;
1714 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1715 RTLIB::FMA_F32,
1716 RTLIB::FMA_F64,
1717 RTLIB::FMA_F80,
1718 RTLIB::FMA_F128,
1719 RTLIB::FMA_PPCF128),
1720 N->getValueType(0), Ops, CallOptions,
1721 SDLoc(N), Chain);
1722 if (IsStrict)
1723 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1724 GetPairElements(Tmp.first, Lo, Hi);
1725}
1726
1727void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1728 SDValue &Hi) {
1729 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1730 RTLIB::MUL_F32,
1731 RTLIB::MUL_F64,
1732 RTLIB::MUL_F80,
1733 RTLIB::MUL_F128,
1734 RTLIB::MUL_PPCF128), Lo, Hi);
1735}
1736
1737void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1738 SDValue &Lo, SDValue &Hi) {
1739 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1740 RTLIB::NEARBYINT_F32,
1741 RTLIB::NEARBYINT_F64,
1742 RTLIB::NEARBYINT_F80,
1743 RTLIB::NEARBYINT_F128,
1744 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1745}
1746
1747void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1748 SDValue &Hi) {
1749 SDLoc dl(N);
1750 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1751 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1752 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1753}
1754
1755void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1756 SDValue &Hi) {
1757 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1758 SDLoc dl(N);
1759 bool IsStrict = N->isStrictFPOpcode();
1760
1761 SDValue Chain;
1762 if (IsStrict) {
1763 // If the expanded type is the same as the input type, just bypass the node.
1764 if (NVT == N->getOperand(1).getValueType()) {
1765 Hi = N->getOperand(1);
1766 Chain = N->getOperand(0);
1767 } else {
1768 // Other we need to extend.
1769 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1770 { N->getOperand(0), N->getOperand(1) });
1771 Chain = Hi.getValue(1);
1772 }
1773 } else {
1774 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1775 }
1776
1777 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
1778
1779 if (IsStrict)
1780 ReplaceValueWith(SDValue(N, 1), Chain);
1781}
1782
1783void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1784 SDValue &Lo, SDValue &Hi) {
1785 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1786 RTLIB::POW_F32, RTLIB::POW_F64,
1787 RTLIB::POW_F80, RTLIB::POW_F128,
1788 RTLIB::POW_PPCF128), Lo, Hi);
1789}
1790
1791void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1792 SDValue &Lo, SDValue &Hi) {
1793 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1794}
1795
1796void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1797 SDValue &Hi) {
1798 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1799}
1800
1801void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1802 SDValue &Lo, SDValue &Hi) {
1803 assert(N->getValueType(0) == MVT::ppcf128 &&
1804 "Logic only correct for ppcf128!");
1805
1806 SDLoc dl(N);
1807 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1808 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1809 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1810}
1811
1812void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1813 SDValue &Lo, SDValue &Hi) {
1814 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1815 RTLIB::REM_F32, RTLIB::REM_F64,
1816 RTLIB::REM_F80, RTLIB::REM_F128,
1817 RTLIB::REM_PPCF128), Lo, Hi);
1818}
1819
1820void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1821 SDValue &Lo, SDValue &Hi) {
1822 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1823 RTLIB::RINT_F32, RTLIB::RINT_F64,
1824 RTLIB::RINT_F80, RTLIB::RINT_F128,
1825 RTLIB::RINT_PPCF128), Lo, Hi);
1826}
1827
1828void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1829 SDValue &Lo, SDValue &Hi) {
1830 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1831 RTLIB::ROUND_F32,
1832 RTLIB::ROUND_F64,
1833 RTLIB::ROUND_F80,
1834 RTLIB::ROUND_F128,
1835 RTLIB::ROUND_PPCF128), Lo, Hi);
1836}
1837
1838void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1839 SDValue &Lo, SDValue &Hi) {
1840 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1841 RTLIB::ROUNDEVEN_F32,
1842 RTLIB::ROUNDEVEN_F64,
1843 RTLIB::ROUNDEVEN_F80,
1844 RTLIB::ROUNDEVEN_F128,
1845 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1846}
1847
1848void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1849 SDValue &Lo, SDValue &Hi) {
1850 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1851 RTLIB::SIN_F32, RTLIB::SIN_F64,
1852 RTLIB::SIN_F80, RTLIB::SIN_F128,
1853 RTLIB::SIN_PPCF128), Lo, Hi);
1854}
1855
1856void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
1857 SDValue &Hi) {
1858 ExpandFloatRes_Unary(N,
1859 GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32,
1860 RTLIB::SINH_F64, RTLIB::SINH_F80,
1861 RTLIB::SINH_F128, RTLIB::SINH_PPCF128),
1862 Lo, Hi);
1863}
1864
1865void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1866 SDValue &Lo, SDValue &Hi) {
1867 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1868 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1869 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1870 RTLIB::SQRT_PPCF128), Lo, Hi);
1871}
1872
1873void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1874 SDValue &Hi) {
1875 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1876 RTLIB::SUB_F32,
1877 RTLIB::SUB_F64,
1878 RTLIB::SUB_F80,
1879 RTLIB::SUB_F128,
1880 RTLIB::SUB_PPCF128), Lo, Hi);
1881}
1882
1883void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
1884 SDValue &Hi) {
1885 ExpandFloatRes_Unary(N,
1886 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32,
1887 RTLIB::TAN_F64, RTLIB::TAN_F80,
1888 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
1889 Lo, Hi);
1890}
1891
1892void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
1893 SDValue &Hi) {
1894 ExpandFloatRes_Unary(N,
1895 GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32,
1896 RTLIB::TANH_F64, RTLIB::TANH_F80,
1897 RTLIB::TANH_F128, RTLIB::TANH_PPCF128),
1898 Lo, Hi);
1899}
1900
1901void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1902 SDValue &Lo, SDValue &Hi) {
1903 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1904 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1905 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1906 RTLIB::TRUNC_PPCF128), Lo, Hi);
1907}
1908
1909void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1910 SDValue &Hi) {
1911 if (ISD::isNormalLoad(N)) {
1912 ExpandRes_NormalLoad(N, Lo, Hi);
1913 return;
1914 }
1915
1916 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1917 LoadSDNode *LD = cast<LoadSDNode>(N);
1918 SDValue Chain = LD->getChain();
1919 SDValue Ptr = LD->getBasePtr();
1920 SDLoc dl(N);
1921
1922 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1923 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1924 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1925
1926 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1927 LD->getMemoryVT(), LD->getMemOperand());
1928
1929 // Remember the chain.
1930 Chain = Hi.getValue(1);
1931
1932 // The low part is zero.
1933 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
1934
1935 // Modified the chain - switch anything that used the old chain to use the
1936 // new one.
1937 ReplaceValueWith(SDValue(LD, 1), Chain);
1938}
1939
1940void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1941 SDValue &Hi) {
1942 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1943 EVT VT = N->getValueType(0);
1944 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1945 bool Strict = N->isStrictFPOpcode();
1946 SDValue Src = N->getOperand(Strict ? 1 : 0);
1947 EVT SrcVT = Src.getValueType();
1948 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1949 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1950 SDLoc dl(N);
1951 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1952
1953 // TODO: Any other flags to propagate?
1954 SDNodeFlags Flags;
1955 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1956
1957 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1958 // When promoting partial word types to i32 we must honor the signedness,
1959 // though.
1960 if (SrcVT.bitsLE(MVT::i32)) {
1961 // The integer can be represented exactly in an f64.
1962 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
1963 if (Strict) {
1964 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1965 {Chain, Src}, Flags);
1966 Chain = Hi.getValue(1);
1967 } else
1968 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1969 } else {
1970 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1971 if (SrcVT.bitsLE(MVT::i64)) {
1972 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1973 MVT::i64, Src);
1974 LC = RTLIB::SINTTOFP_I64_PPCF128;
1975 } else if (SrcVT.bitsLE(MVT::i128)) {
1976 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1977 LC = RTLIB::SINTTOFP_I128_PPCF128;
1978 }
1979 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1980
1981 TargetLowering::MakeLibCallOptions CallOptions;
1982 CallOptions.setSExt(true);
1983 std::pair<SDValue, SDValue> Tmp =
1984 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1985 if (Strict)
1986 Chain = Tmp.second;
1987 GetPairElements(Tmp.first, Lo, Hi);
1988 }
1989
1990 // No need to complement for unsigned 32-bit integers
1991 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1992 if (Strict)
1993 ReplaceValueWith(SDValue(N, 1), Chain);
1994
1995 return;
1996 }
1997
1998 // Unsigned - fix up the SINT_TO_FP value just calculated.
1999 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2000 // keep semantics correctness if the integer is not exactly representable
2001 // here. See ExpandLegalINT_TO_FP.
2002 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
2003 SrcVT = Src.getValueType();
2004
2005 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2006 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2007 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2008 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2009 ArrayRef<uint64_t> Parts;
2010
2011 switch (SrcVT.getSimpleVT().SimpleTy) {
2012 default:
2013 llvm_unreachable("Unsupported UINT_TO_FP!");
2014 case MVT::i32:
2015 Parts = TwoE32;
2016 break;
2017 case MVT::i64:
2018 Parts = TwoE64;
2019 break;
2020 case MVT::i128:
2021 Parts = TwoE128;
2022 break;
2023 }
2024
2025 // TODO: Are there other fast-math-flags to propagate to this FADD?
2026 SDValue NewLo = DAG.getConstantFP(
2027 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
2028 if (Strict) {
2029 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
2030 {Chain, Hi, NewLo}, Flags);
2031 Chain = Lo.getValue(1);
2032 ReplaceValueWith(SDValue(N, 1), Chain);
2033 } else
2034 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
2035 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
2036 Lo, Hi, ISD::SETLT);
2037 GetPairElements(Lo, Lo, Hi);
2038}
2039
2040
2041//===----------------------------------------------------------------------===//
2042// Float Operand Expansion
2043//===----------------------------------------------------------------------===//
2044
2045/// ExpandFloatOperand - This method is called when the specified operand of the
2046/// specified node is found to need expansion. At this point, all of the result
2047/// types of the node are known to be legal, but other operands of the node may
2048/// need promotion or expansion as well as the specified one.
2049bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2050 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2051 SDValue Res = SDValue();
2052
2053 // See if the target wants to custom expand this node.
2054 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2055 return false;
2056
2057 switch (N->getOpcode()) {
2058 default:
2059#ifndef NDEBUG
2060 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2061 N->dump(&DAG); dbgs() << "\n";
2062#endif
2063 report_fatal_error("Do not know how to expand this operator's operand!");
2064
2065 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2066 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2067 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2068
2069 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2070 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2071 case ISD::STRICT_FP_ROUND:
2072 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2073 case ISD::STRICT_FP_TO_SINT:
2074 case ISD::STRICT_FP_TO_UINT:
2075 case ISD::FP_TO_SINT:
2076 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2077 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2078 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2079 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2080 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2081 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2082 case ISD::STRICT_FSETCC:
2083 case ISD::STRICT_FSETCCS:
2084 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2085 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
2086 OpNo); break;
2087 }
2088
2089 // If the result is null, the sub-method took care of registering results etc.
2090 if (!Res.getNode()) return false;
2091
2092 // If the result is N, the sub-method updated N in place. Tell the legalizer
2093 // core about this.
2094 if (Res.getNode() == N)
2095 return true;
2096
2097 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2098 "Invalid operand expansion");
2099
2100 ReplaceValueWith(SDValue(N, 0), Res);
2101 return false;
2102}
2103
2104/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2105/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2106void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2107 SDValue &NewRHS,
2108 ISD::CondCode &CCCode,
2109 const SDLoc &dl, SDValue &Chain,
2110 bool IsSignaling) {
2111 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2112 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2113 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2114
2115 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2116
2117 // FIXME: This generated code sucks. We want to generate
2118 // FCMPU crN, hi1, hi2
2119 // BNE crN, L:
2120 // FCMPU crN, lo1, lo2
2121 // The following can be improved, but not that much.
2122 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2123 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2124 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
2125 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2126 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2127 RHSLo, CCCode, OutputChain, IsSignaling);
2128 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2129 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2130 Tmp1 =
2131 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
2132 ISD::SETUNE, OutputChain, IsSignaling);
2133 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2134 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2135 RHSHi, CCCode, OutputChain, IsSignaling);
2136 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2137 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2138 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
2139 NewRHS = SDValue(); // LHS is the result, not a compare.
2140 Chain = OutputChain;
2141}
2142
2143SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2144 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2145 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2146 SDValue Chain;
2147 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2148
2149 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2150 // against zero to select between true and false values.
2151 if (!NewRHS.getNode()) {
2152 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2153 CCCode = ISD::SETNE;
2154 }
2155
2156 // Update N to have the operands specified.
2157 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2158 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2159 N->getOperand(4)), 0);
2160}
2161
2162SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2163 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2164 "Logic only correct for ppcf128!");
2165 SDValue Lo, Hi;
2166 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2167 // The ppcf128 value is providing only the sign; take it from the
2168 // higher-order double (which must have the larger magnitude).
2169 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2170 N->getValueType(0), N->getOperand(0), Hi);
2171}
2172
2173SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2174 bool IsStrict = N->isStrictFPOpcode();
2175 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2176 "Logic only correct for ppcf128!");
2177 SDValue Lo, Hi;
2178 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2179
2180 if (!IsStrict)
2181 // Round it the rest of the way (e.g. to f32) if needed.
2182 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2183 N->getValueType(0), Hi, N->getOperand(1));
2184
2185 // Eliminate the node if the input float type is the same as the output float
2186 // type.
2187 if (Hi.getValueType() == N->getValueType(0)) {
2188 // Connect the output chain to the input chain, unlinking the node.
2189 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2190 ReplaceValueWith(SDValue(N, 0), Hi);
2191 return SDValue();
2192 }
2193
2194 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2195 {N->getValueType(0), MVT::Other},
2196 {N->getOperand(0), Hi, N->getOperand(2)});
2197 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2198 ReplaceValueWith(SDValue(N, 0), Expansion);
2199 return SDValue();
2200}
2201
2202SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2203 EVT RVT = N->getValueType(0);
2204 SDLoc dl(N);
2205
2206 bool IsStrict = N->isStrictFPOpcode();
2207 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2208 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2209 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2210 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2211
2212 EVT NVT;
2213 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2214 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2215 "Unsupported FP_TO_XINT!");
2216 TargetLowering::MakeLibCallOptions CallOptions;
2217 std::pair<SDValue, SDValue> Tmp =
2218 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2219 if (!IsStrict)
2220 return Tmp.first;
2221
2222 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2223 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2224 return SDValue();
2225}
2226
2227SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2228 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2229 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2230 SDValue Chain;
2231 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2232
2233 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2234 // against zero to select between true and false values.
2235 if (!NewRHS.getNode()) {
2236 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2237 CCCode = ISD::SETNE;
2238 }
2239
2240 // Update N to have the operands specified.
2241 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2242 N->getOperand(2), N->getOperand(3),
2243 DAG.getCondCode(CCCode)), 0);
2244}
2245
2246SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2247 bool IsStrict = N->isStrictFPOpcode();
2248 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2249 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2250 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2251 ISD::CondCode CCCode =
2252 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2253 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2254 N->getOpcode() == ISD::STRICT_FSETCCS);
2255
2256 // FloatExpandSetCCOperands always returned a scalar.
2257 assert(!NewRHS.getNode() && "Expect to return scalar");
2258 assert(NewLHS.getValueType() == N->getValueType(0) &&
2259 "Unexpected setcc expansion!");
2260 if (Chain) {
2261 ReplaceValueWith(SDValue(N, 0), NewLHS);
2262 ReplaceValueWith(SDValue(N, 1), Chain);
2263 return SDValue();
2264 }
2265 return NewLHS;
2266}
2267
2268SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2269 if (ISD::isNormalStore(N))
2270 return ExpandOp_NormalStore(N, OpNo);
2271
2272 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2273 assert(OpNo == 1 && "Can only expand the stored value so far");
2274 StoreSDNode *ST = cast<StoreSDNode>(N);
2275
2276 SDValue Chain = ST->getChain();
2277 SDValue Ptr = ST->getBasePtr();
2278
2279 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2280 ST->getValue().getValueType());
2281 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2282 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2283 (void)NVT;
2284
2285 SDValue Lo, Hi;
2286 GetExpandedOp(ST->getValue(), Lo, Hi);
2287
2288 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2289 ST->getMemoryVT(), ST->getMemOperand());
2290}
2291
2292SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2293 EVT RVT = N->getValueType(0);
2294 EVT RetVT = N->getOperand(0).getValueType();
2295 TargetLowering::MakeLibCallOptions CallOptions;
2296 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2297 RTLIB::LROUND_F32,
2298 RTLIB::LROUND_F64,
2299 RTLIB::LROUND_F80,
2300 RTLIB::LROUND_F128,
2301 RTLIB::LROUND_PPCF128),
2302 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2303}
2304
2305SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2306 EVT RVT = N->getValueType(0);
2307 EVT RetVT = N->getOperand(0).getValueType();
2308 TargetLowering::MakeLibCallOptions CallOptions;
2309 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2310 RTLIB::LLROUND_F32,
2311 RTLIB::LLROUND_F64,
2312 RTLIB::LLROUND_F80,
2313 RTLIB::LLROUND_F128,
2314 RTLIB::LLROUND_PPCF128),
2315 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2316}
2317
2318SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2319 EVT RVT = N->getValueType(0);
2320 EVT RetVT = N->getOperand(0).getValueType();
2321 TargetLowering::MakeLibCallOptions CallOptions;
2322 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2323 RTLIB::LRINT_F32,
2324 RTLIB::LRINT_F64,
2325 RTLIB::LRINT_F80,
2326 RTLIB::LRINT_F128,
2327 RTLIB::LRINT_PPCF128),
2328 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2329}
2330
2331SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2332 EVT RVT = N->getValueType(0);
2333 EVT RetVT = N->getOperand(0).getValueType();
2334 TargetLowering::MakeLibCallOptions CallOptions;
2335 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2336 RTLIB::LLRINT_F32,
2337 RTLIB::LLRINT_F64,
2338 RTLIB::LLRINT_F80,
2339 RTLIB::LLRINT_F128,
2340 RTLIB::LLRINT_PPCF128),
2341 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2342}
2343
2344//===----------------------------------------------------------------------===//
2345// Float Operand Promotion
2346//===----------------------------------------------------------------------===//
2347//
2348
2349static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2350 if (OpVT == MVT::f16) {
2351 return ISD::FP16_TO_FP;
2352 } else if (RetVT == MVT::f16) {
2353 return ISD::FP_TO_FP16;
2354 } else if (OpVT == MVT::bf16) {
2355 return ISD::BF16_TO_FP;
2356 } else if (RetVT == MVT::bf16) {
2357 return ISD::FP_TO_BF16;
2358 }
2359
2360 report_fatal_error("Attempt at an invalid promotion-related conversion");
2361}
2362
2363static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2364 if (OpVT == MVT::f16)
2365 return ISD::STRICT_FP16_TO_FP;
2366
2367 if (RetVT == MVT::f16)
2368 return ISD::STRICT_FP_TO_FP16;
2369
2370 if (OpVT == MVT::bf16)
2371 return ISD::STRICT_BF16_TO_FP;
2372
2373 if (RetVT == MVT::bf16)
2374 return ISD::STRICT_FP_TO_BF16;
2375
2376 report_fatal_error("Attempt at an invalid promotion-related conversion");
2377}
2378
2379bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2380 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2381 SDValue R = SDValue();
2382
2383 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2384 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2385 return false;
2386 }
2387
2388 // Nodes that use a promotion-requiring floating point operand, but doesn't
2389 // produce a promotion-requiring floating point result, need to be legalized
2390 // to use the promoted float operand. Nodes that produce at least one
2391 // promotion-requiring floating point result have their operands legalized as
2392 // a part of PromoteFloatResult.
2393 // clang-format off
2394 switch (N->getOpcode()) {
2395 default:
2396 #ifndef NDEBUG
2397 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2398 N->dump(&DAG); dbgs() << "\n";
2399 #endif
2400 report_fatal_error("Do not know how to promote this operator's operand!");
2401
2402 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2403 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2404 case ISD::FP_TO_SINT:
2405 case ISD::FP_TO_UINT:
2406 case ISD::LRINT:
2407 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2408 case ISD::FP_TO_SINT_SAT:
2409 case ISD::FP_TO_UINT_SAT:
2410 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2411 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2412 case ISD::STRICT_FP_EXTEND:
2413 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2414 break;
2415 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2416 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2417 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2418 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2419 }
2420 // clang-format on
2421
2422 if (R.getNode())
2423 ReplaceValueWith(SDValue(N, 0), R);
2424 return false;
2425}
2426
2427SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2428 SDValue Op = N->getOperand(0);
2429 EVT OpVT = Op->getValueType(0);
2430
2431 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2432 EVT PromotedVT = Promoted->getValueType(0);
2433
2434 // Convert the promoted float value to the desired IVT.
2435 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2436 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2437 IVT, Promoted);
2438 // The final result type might not be an scalar so we need a bitcast. The
2439 // bitcast will be further legalized if needed.
2440 return DAG.getBitcast(N->getValueType(0), Convert);
2441}
2442
2443// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2444// PromoteFloatRes_FCOPYSIGN.
2445SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2446 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2447 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2448
2449 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2450 N->getOperand(0), Op1);
2451}
2452
2453// Convert the promoted float value to the desired integer type
2454SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2455 SDValue Op = GetPromotedFloat(N->getOperand(0));
2456 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2457}
2458
2459SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2460 unsigned OpNo) {
2461 SDValue Op = GetPromotedFloat(N->getOperand(0));
2462 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2463 N->getOperand(1));
2464}
2465
2466SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2467 SDValue Op = GetPromotedFloat(N->getOperand(0));
2468 EVT VT = N->getValueType(0);
2469
2470 // Desired VT is same as promoted type. Use promoted float directly.
2471 if (VT == Op->getValueType(0))
2472 return Op;
2473
2474 // Else, extend the promoted float value to the desired VT.
2475 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2476}
2477
2478SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2479 unsigned OpNo) {
2480 assert(OpNo == 1 && "Promoting unpromotable operand");
2481
2482 SDValue Op = GetPromotedFloat(N->getOperand(1));
2483 EVT VT = N->getValueType(0);
2484
2485 // Desired VT is same as promoted type. Use promoted float directly.
2486 if (VT == Op->getValueType(0)) {
2487 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2488 return Op;
2489 }
2490
2491 // Else, extend the promoted float value to the desired VT.
2492 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2493 N->getOperand(0), Op);
2494 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2495 return Res;
2496}
2497
2498// Promote the float operands used for comparison. The true- and false-
2499// operands have the same type as the result and are promoted, if needed, by
2500// PromoteFloatRes_SELECT_CC
2501SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2502 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2503 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2504
2505 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2506 LHS, RHS, N->getOperand(2), N->getOperand(3),
2507 N->getOperand(4));
2508}
2509
2510// Construct a SETCC that compares the promoted values and sets the conditional
2511// code.
2512SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2513 EVT VT = N->getValueType(0);
2514 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2515 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2516 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2517
2518 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2519
2520}
2521
2522// Lower the promoted Float down to the integer value of same size and construct
2523// a STORE of the integer value.
2524SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2525 StoreSDNode *ST = cast<StoreSDNode>(N);
2526 SDValue Val = ST->getValue();
2527 SDLoc DL(N);
2528
2529 SDValue Promoted = GetPromotedFloat(Val);
2530 EVT VT = ST->getOperand(1).getValueType();
2531 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2532
2533 SDValue NewVal;
2534 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2535 IVT, Promoted);
2536
2537 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2538 ST->getMemOperand());
2539}
2540
2541SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2542 unsigned OpNo) {
2543 AtomicSDNode *ST = cast<AtomicSDNode>(N);
2544 SDValue Val = ST->getVal();
2545 SDLoc DL(N);
2546
2547 SDValue Promoted = GetPromotedFloat(Val);
2548 EVT VT = ST->getOperand(1).getValueType();
2549 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2550
2551 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
2552 DL, IVT, Promoted);
2553
2554 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
2555 ST->getBasePtr(), ST->getMemOperand());
2556}
2557
2558//===----------------------------------------------------------------------===//
2559// Float Result Promotion
2560//===----------------------------------------------------------------------===//
2561
2562void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2563 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2564 SDValue R = SDValue();
2565
2566 // See if the target wants to custom expand this node.
2567 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2568 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2569 return;
2570 }
2571
2572 switch (N->getOpcode()) {
2573 // These opcodes cannot appear if promotion of FP16 is done in the backend
2574 // instead of Clang
2575 case ISD::FP16_TO_FP:
2576 case ISD::FP_TO_FP16:
2577 default:
2578#ifndef NDEBUG
2579 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2580 N->dump(&DAG); dbgs() << "\n";
2581#endif
2582 report_fatal_error("Do not know how to promote this operator's result!");
2583
2584 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2585 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2586 case ISD::EXTRACT_VECTOR_ELT:
2587 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2588 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2589
2590 // Unary FP Operations
2591 case ISD::FABS:
2592 case ISD::FACOS:
2593 case ISD::FASIN:
2594 case ISD::FATAN:
2595 case ISD::FCBRT:
2596 case ISD::FCEIL:
2597 case ISD::FCOS:
2598 case ISD::FCOSH:
2599 case ISD::FEXP:
2600 case ISD::FEXP2:
2601 case ISD::FEXP10:
2602 case ISD::FFLOOR:
2603 case ISD::FLOG:
2604 case ISD::FLOG2:
2605 case ISD::FLOG10:
2606 case ISD::FNEARBYINT:
2607 case ISD::FNEG:
2608 case ISD::FRINT:
2609 case ISD::FROUND:
2610 case ISD::FROUNDEVEN:
2611 case ISD::FSIN:
2612 case ISD::FSINH:
2613 case ISD::FSQRT:
2614 case ISD::FTRUNC:
2615 case ISD::FTAN:
2616 case ISD::FTANH:
2617 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2618
2619 // Binary FP Operations
2620 case ISD::FADD:
2621 case ISD::FDIV:
2622 case ISD::FMAXIMUM:
2623 case ISD::FMINIMUM:
2624 case ISD::FMAXNUM:
2625 case ISD::FMINNUM:
2626 case ISD::FMAXNUM_IEEE:
2627 case ISD::FMINNUM_IEEE:
2628 case ISD::FMUL:
2629 case ISD::FPOW:
2630 case ISD::FREM:
2631 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2632
2633 case ISD::FMA: // FMA is same as FMAD
2634 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2635
2636 case ISD::FPOWI:
2637 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2638 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2639
2640 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2641 case ISD::STRICT_FP_ROUND:
2642 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2643 break;
2644 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2645 case ISD::ATOMIC_LOAD:
2646 R = PromoteFloatRes_ATOMIC_LOAD(N);
2647 break;
2648 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2649 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2650
2651 case ISD::SINT_TO_FP:
2652 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2653 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2654 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2655 case ISD::VECREDUCE_FADD:
2656 case ISD::VECREDUCE_FMUL:
2657 case ISD::VECREDUCE_FMIN:
2658 case ISD::VECREDUCE_FMAX:
2659 case ISD::VECREDUCE_FMAXIMUM:
2660 case ISD::VECREDUCE_FMINIMUM:
2661 R = PromoteFloatRes_VECREDUCE(N);
2662 break;
2663 case ISD::VECREDUCE_SEQ_FADD:
2664 case ISD::VECREDUCE_SEQ_FMUL:
2665 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2666 break;
2667 }
2668
2669 if (R.getNode())
2670 SetPromotedFloat(SDValue(N, ResNo), R);
2671}
2672
2673// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2674// At this point, it is not possible to determine if the bitcast value is
2675// eventually stored to memory or promoted to f32 or promoted to a floating
2676// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2677// STORE promotion handlers.
2678SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2679 EVT VT = N->getValueType(0);
2680 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2681 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2682 // bitcast will be legalized further if necessary.
2683 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2684 N->getOperand(0).getValueType().getSizeInBits());
2685 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2686 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2687}
2688
2689SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2690 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2691 EVT VT = N->getValueType(0);
2692 SDLoc DL(N);
2693
2694 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2695 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2696 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2697 IVT);
2698
2699 // Convert the Constant to the desired FP type
2700 // FIXME We might be able to do the conversion during compilation and get rid
2701 // of it from the object code
2702 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2703 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2704}
2705
2706// If the Index operand is a constant, try to redirect the extract operation to
2707// the correct legalized vector. If not, bit-convert the input vector to
2708// equivalent integer vector. Extract the element as an (bit-cast) integer
2709// value and convert it to the promoted type.
2710SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2711 SDLoc DL(N);
2712
2713 // If the index is constant, try to extract the value from the legalized
2714 // vector type.
2715 if (isa<ConstantSDNode>(N->getOperand(1))) {
2716 SDValue Vec = N->getOperand(0);
2717 SDValue Idx = N->getOperand(1);
2718 EVT VecVT = Vec->getValueType(0);
2719 EVT EltVT = VecVT.getVectorElementType();
2720
2721 uint64_t IdxVal = Idx->getAsZExtVal();
2722
2723 switch (getTypeAction(VecVT)) {
2724 default: break;
2725 case TargetLowering::TypeScalarizeVector: {
2726 SDValue Res = GetScalarizedVector(N->getOperand(0));
2727 ReplaceValueWith(SDValue(N, 0), Res);
2728 return SDValue();
2729 }
2730 case TargetLowering::TypeWidenVector: {
2731 Vec = GetWidenedVector(Vec);
2732 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2733 ReplaceValueWith(SDValue(N, 0), Res);
2734 return SDValue();
2735 }
2736 case TargetLowering::TypeSplitVector: {
2737 SDValue Lo, Hi;
2738 GetSplitVector(Vec, Lo, Hi);
2739
2740 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2741 SDValue Res;
2742 if (IdxVal < LoElts)
2743 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2744 else
2745 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2746 DAG.getConstant(IdxVal - LoElts, DL,
2747 Idx.getValueType()));
2748 ReplaceValueWith(SDValue(N, 0), Res);
2749 return SDValue();
2750 }
2751
2752 }
2753 }
2754
2755 // Bit-convert the input vector to the equivalent integer vector
2756 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2757 EVT IVT = NewOp.getValueType().getVectorElementType();
2758
2759 // Extract the element as an (bit-cast) integer value
2760 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2761 NewOp, N->getOperand(1));
2762
2763 // Convert the element to the desired FP type
2764 EVT VT = N->getValueType(0);
2765 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2766 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2767}
2768
2769// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2770// needs promotion, so does the argument X. Note that Y, if needed, will be
2771// handled during operand promotion.
2772SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2773 EVT VT = N->getValueType(0);
2774 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2775 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2776
2777 SDValue Op1 = N->getOperand(1);
2778
2779 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2780}
2781
2782// Unary operation where the result and the operand have PromoteFloat type
2783// action. Construct a new SDNode with the promoted float value of the old
2784// operand.
2785SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2786 EVT VT = N->getValueType(0);
2787 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2788 SDValue Op = GetPromotedFloat(N->getOperand(0));
2789
2790 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2791}
2792
2793// Binary operations where the result and both operands have PromoteFloat type
2794// action. Construct a new SDNode with the promoted float values of the old
2795// operands.
2796SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2797 EVT VT = N->getValueType(0);
2798 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2799 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2800 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2801 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2802}
2803
2804SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2805 EVT VT = N->getValueType(0);
2806 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2807 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2808 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2809 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2810
2811 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2812}
2813
2814// Promote the Float (first) operand and retain the Integer (second) operand
2815SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2816 EVT VT = N->getValueType(0);
2817 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2818 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2819 SDValue Op1 = N->getOperand(1);
2820
2821 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2822}
2823
2824SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2825 EVT VT = N->getValueType(0);
2826 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2827 SDValue Op = GetPromotedFloat(N->getOperand(0));
2828 SDValue Res =
2829 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2830
2831 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2832 return Res;
2833}
2834
2835// Explicit operation to reduce precision. Reduce the value to half precision
2836// and promote it back to the legal type.
2837SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2838 SDLoc DL(N);
2839
2840 SDValue Op = N->getOperand(0);
2841 EVT VT = N->getValueType(0);
2842 EVT OpVT = Op->getValueType(0);
2843 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2844 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2845
2846 // Round promoted float to desired precision
2847 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2848 // Promote it back to the legal output type
2849 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2850}
2851
2852// Explicit operation to reduce precision. Reduce the value to half precision
2853// and promote it back to the legal type.
2854SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2855 SDLoc DL(N);
2856
2857 SDValue Chain = N->getOperand(0);
2858 SDValue Op = N->getOperand(1);
2859 EVT VT = N->getValueType(0);
2860 EVT OpVT = Op->getValueType(0);
2861 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2862 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2863
2864 // Round promoted float to desired precision
2865 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2866 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2867 // Promote it back to the legal output type
2868 SDValue Res =
2869 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2870 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2871 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2872 return Res;
2873}
2874
2875SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2876 LoadSDNode *L = cast<LoadSDNode>(N);
2877 EVT VT = N->getValueType(0);
2878
2879 // Load the value as an integer value with the same number of bits.
2880 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2881 SDValue newL = DAG.getLoad(
2882 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2883 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2884 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2885 // Legalize the chain result by replacing uses of the old value chain with the
2886 // new one
2887 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2888
2889 // Convert the integer value to the desired FP type
2890 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2891 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2892}
2893
2894SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
2895 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2896 EVT VT = AM->getValueType(0);
2897
2898 // Load the value as an integer value with the same number of bits.
2899 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2900 SDValue newL = DAG.getAtomic(
2901 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other),
2902 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
2903
2904 // Legalize the chain result by replacing uses of the old value chain with the
2905 // new one
2906 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2907
2908 // Convert the integer value to the desired FP type
2909 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2910 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL);
2911}
2912
2913// Construct a new SELECT node with the promoted true- and false- values.
2914SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2915 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2916 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2917
2918 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2919 N->getOperand(0), TrueVal, FalseVal);
2920}
2921
2922// Construct a new SELECT_CC node with the promoted true- and false- values.
2923// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2924SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2925 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2926 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2927
2928 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2929 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2930 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2931}
2932
2933// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2934// float type.
2935SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2936 SDLoc DL(N);
2937 EVT VT = N->getValueType(0);
2938 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2939 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2940 // Round the value to the desired precision (that of the source type).
2941 return DAG.getNode(
2942 ISD::FP_EXTEND, DL, NVT,
2943 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2944 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2945}
2946
2947SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2948 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2949 N->getValueType(0)));
2950}
2951
2952SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2953 // Expand and promote recursively.
2954 // TODO: This is non-optimal, but dealing with the concurrently happening
2955 // vector-legalization is non-trivial. We could do something similar to
2956 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2957 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2958 return SDValue();
2959}
2960
2961SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2962 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2963 return SDValue();
2964}
2965
2966SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2967 EVT VT = N->getValueType(0);
2968
2969 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2970 SDLoc SL(N);
2971
2972 SDValue CastVal = BitConvertToInteger(AM->getVal());
2973 EVT CastVT = CastVal.getValueType();
2974
2975 SDValue NewAtomic
2976 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2977 DAG.getVTList(CastVT, MVT::Other),
2978 { AM->getChain(), AM->getBasePtr(), CastVal },
2979 AM->getMemOperand());
2980
2981 SDValue Result = NewAtomic;
2982
2983 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2984 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2985 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2986 NewAtomic);
2987 }
2988
2989 // Legalize the chain result by replacing uses of the old value chain with the
2990 // new one
2991 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2992
2993 return Result;
2994
2995}
2996
2997//===----------------------------------------------------------------------===//
2998// Half Result Soft Promotion
2999//===----------------------------------------------------------------------===//
3000
3001void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
3002 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
3003 N->dump(&DAG));
3004 SDValue R = SDValue();
3005
3006 // See if the target wants to custom expand this node.
3007 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
3008 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
3009 return;
3010 }
3011
3012 switch (N->getOpcode()) {
3013 default:
3014#ifndef NDEBUG
3015 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
3016 N->dump(&DAG); dbgs() << "\n";
3017#endif
3018 report_fatal_error("Do not know how to soft promote this operator's "
3019 "result!");
3020
3021 case ISD::ARITH_FENCE:
3022 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
3023 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
3024 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
3025 case ISD::EXTRACT_VECTOR_ELT:
3026 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
3027 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
3028 case ISD::STRICT_FP_ROUND:
3029 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
3030
3031 // Unary FP Operations
3032 case ISD::FABS:
3033 case ISD::FACOS:
3034 case ISD::FASIN:
3035 case ISD::FATAN:
3036 case ISD::FCBRT:
3037 case ISD::FCEIL:
3038 case ISD::FCOS:
3039 case ISD::FCOSH:
3040 case ISD::FEXP:
3041 case ISD::FEXP2:
3042 case ISD::FEXP10:
3043 case ISD::FFLOOR:
3044 case ISD::FLOG:
3045 case ISD::FLOG2:
3046 case ISD::FLOG10:
3047 case ISD::FNEARBYINT:
3048 case ISD::FNEG:
3049 case ISD::FREEZE:
3050 case ISD::FRINT:
3051 case ISD::FROUND:
3052 case ISD::FROUNDEVEN:
3053 case ISD::FSIN:
3054 case ISD::FSINH:
3055 case ISD::FSQRT:
3056 case ISD::FTRUNC:
3057 case ISD::FTAN:
3058 case ISD::FTANH:
3059 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
3060
3061 // Binary FP Operations
3062 case ISD::FADD:
3063 case ISD::FDIV:
3064 case ISD::FMAXIMUM:
3065 case ISD::FMINIMUM:
3066 case ISD::FMAXNUM:
3067 case ISD::FMINNUM:
3068 case ISD::FMUL:
3069 case ISD::FPOW:
3070 case ISD::FREM:
3071 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
3072
3073 case ISD::FMA: // FMA is same as FMAD
3074 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
3075
3076 case ISD::FPOWI:
3077 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
3078
3079 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
3080
3081 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
3082 case ISD::ATOMIC_LOAD:
3083 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
3084 break;
3085 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
3086 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
3087 case ISD::SINT_TO_FP:
3088 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
3089 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
3090 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
3091 case ISD::VECREDUCE_FADD:
3092 case ISD::VECREDUCE_FMUL:
3093 case ISD::VECREDUCE_FMIN:
3094 case ISD::VECREDUCE_FMAX:
3095 case ISD::VECREDUCE_FMAXIMUM:
3096 case ISD::VECREDUCE_FMINIMUM:
3097 R = SoftPromoteHalfRes_VECREDUCE(N);
3098 break;
3099 case ISD::VECREDUCE_SEQ_FADD:
3100 case ISD::VECREDUCE_SEQ_FMUL:
3101 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
3102 break;
3103 }
3104
3105 if (R.getNode())
3106 SetSoftPromotedHalf(SDValue(N, ResNo), R);
3107}
3108
3109SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
3110 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16,
3111 BitConvertToInteger(N->getOperand(0)));
3112}
3113
3114SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
3115 return BitConvertToInteger(N->getOperand(0));
3116}
3117
3118SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
3119 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
3120
3121 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3122 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
3123 MVT::i16);
3124}
3125
3126SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3127 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3128 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3129 NewOp.getValueType().getVectorElementType(), NewOp,
3130 N->getOperand(1));
3131}
3132
3133SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3134 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
3135 SDValue RHS = BitConvertToInteger(N->getOperand(1));
3136 SDLoc dl(N);
3137
3138 EVT LVT = LHS.getValueType();
3139 EVT RVT = RHS.getValueType();
3140
3141 unsigned LSize = LVT.getSizeInBits();
3142 unsigned RSize = RVT.getSizeInBits();
3143
3144 // First get the sign bit of second operand.
3145 SDValue SignBit = DAG.getNode(
3146 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3147 DAG.getConstant(RSize - 1, dl,
3148 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3149 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
3150
3151 // Shift right or sign-extend it if the two operands have different types.
3152 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3153 if (SizeDiff > 0) {
3154 SignBit =
3155 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
3156 DAG.getConstant(SizeDiff, dl,
3157 TLI.getShiftAmountTy(SignBit.getValueType(),
3158 DAG.getDataLayout())));
3159 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
3160 } else if (SizeDiff < 0) {
3161 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
3162 SignBit =
3163 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
3164 DAG.getConstant(-SizeDiff, dl,
3165 TLI.getShiftAmountTy(SignBit.getValueType(),
3166 DAG.getDataLayout())));
3167 }
3168
3169 // Clear the sign bit of the first operand.
3170 SDValue Mask = DAG.getNode(
3171 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3172 DAG.getConstant(LSize - 1, dl,
3173 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3174 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3175 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
3176
3177 // Or the value with the sign bit.
3178 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
3179}
3180
3181SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3182 EVT OVT = N->getValueType(0);
3183 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3184 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3185 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3186 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3187 SDLoc dl(N);
3188
3189 // Promote to the larger FP type.
3190 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3191 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3192 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3193 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
3194
3195 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
3196
3197 // Convert back to FP16 as an integer.
3198 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3199}
3200
3201SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3202 EVT OVT = N->getValueType(0);
3203 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3204 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3205 SDValue Op1 = N->getOperand(1);
3206 SDLoc dl(N);
3207
3208 // Promote to the larger FP type.
3209 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
3210
3211 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3212
3213 // Convert back to FP16 as an integer.
3214 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3215}
3216
3217SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3218 EVT OVT = N->getValueType(0);
3219 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3220 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3221 SDLoc dl(N);
3222
3223 // Promote to the larger FP type.
3224 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3225
3226 SDValue Res = DAG.getNode(N->getOpcode(), dl,
3227 DAG.getVTList(NVT, N->getValueType(1)), Op);
3228
3229 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3230
3231 // Convert back to FP16 as an integer.
3232 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3233}
3234
3235SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3236 EVT RVT = N->getValueType(0);
3237 EVT SVT = N->getOperand(0).getValueType();
3238
3239 if (N->isStrictFPOpcode()) {
3240 // FIXME: assume we only have two f16 variants for now.
3241 unsigned Opcode;
3242 if (RVT == MVT::f16)
3243 Opcode = ISD::STRICT_FP_TO_FP16;
3244 else if (RVT == MVT::bf16)
3245 Opcode = ISD::STRICT_FP_TO_BF16;
3246 else
3247 llvm_unreachable("unknown half type");
3248 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3249 {N->getOperand(0), N->getOperand(1)});
3250 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3251 return Res;
3252 }
3253
3254 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3255 N->getOperand(0));
3256}
3257
3258SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3259 LoadSDNode *L = cast<LoadSDNode>(N);
3260
3261 // Load the value as an integer value with the same number of bits.
3262 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3263 SDValue NewL =
3264 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3265 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3266 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3267 L->getMemOperand()->getFlags(), L->getAAInfo());
3268 // Legalize the chain result by replacing uses of the old value chain with the
3269 // new one
3270 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3271 return NewL;
3272}
3273
3274SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3275 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3276
3277 // Load the value as an integer value with the same number of bits.
3278 SDValue NewL = DAG.getAtomic(
3279 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3280 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3281
3282 // Legalize the chain result by replacing uses of the old value chain with the
3283 // new one
3284 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3285 return NewL;
3286}
3287
3288SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3289 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3290 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3291 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3292 Op2);
3293}
3294
3295SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3296 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3297 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3298 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3299 N->getOperand(0), N->getOperand(1), Op2, Op3,
3300 N->getOperand(4));
3301}
3302
3303SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3304 EVT OVT = N->getValueType(0);
3305 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3306 SDLoc dl(N);
3307
3308 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3309
3310 // Round the value to the softened type.
3311 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3312}
3313
3314SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3315 return DAG.getUNDEF(MVT::i16);
3316}
3317
3318SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3319 EVT OVT = N->getValueType(0);
3320 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3321 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3322 SDLoc dl(N);
3323
3324 // Promote to the larger FP type.
3325 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3326
3327 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3328
3329 // Convert back to FP16 as an integer.
3330 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3331}
3332
3333SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3334 EVT OVT = N->getValueType(0);
3335 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3336 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3337 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3338 SDLoc dl(N);
3339
3340 // Promote to the larger FP type.
3341 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3342 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3343 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3344
3345 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3346
3347 // Convert back to FP16 as an integer.
3348 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3349}
3350
3351SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3352 // Expand and soften recursively.
3353 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3354 return SDValue();
3355}
3356
3357SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3358 // Expand and soften.
3359 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3360 return SDValue();
3361}
3362
3363//===----------------------------------------------------------------------===//
3364// Half Operand Soft Promotion
3365//===----------------------------------------------------------------------===//
3366
3367bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3368 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3369 N->dump(&DAG));
3370 SDValue Res = SDValue();
3371
3372 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3373 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3374 return false;
3375 }
3376
3377 // Nodes that use a promotion-requiring floating point operand, but doesn't
3378 // produce a soft promotion-requiring floating point result, need to be
3379 // legalized to use the soft promoted float operand. Nodes that produce at
3380 // least one soft promotion-requiring floating point result have their
3381 // operands legalized as a part of PromoteFloatResult.
3382 switch (N->getOpcode()) {
3383 default:
3384 #ifndef NDEBUG
3385 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3386 N->dump(&DAG); dbgs() << "\n";
3387 #endif
3388 report_fatal_error("Do not know how to soft promote this operator's "
3389 "operand!");
3390
3391 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3392 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3393 case ISD::FP_TO_SINT:
3394 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3395 case ISD::FP_TO_SINT_SAT:
3396 case ISD::FP_TO_UINT_SAT:
3397 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3398 case ISD::STRICT_FP_EXTEND:
3399 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3400 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3401 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3402 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3403 case ISD::ATOMIC_STORE:
3404 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3405 break;
3406 case ISD::STACKMAP:
3407 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3408 break;
3409 case ISD::PATCHPOINT:
3410 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3411 break;
3412 }
3413
3414 if (!Res.getNode())
3415 return false;
3416
3417 assert(Res.getNode() != N && "Expected a new node!");
3418
3419 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3420 "Invalid operand expansion");
3421
3422 ReplaceValueWith(SDValue(N, 0), Res);
3423 return false;
3424}
3425
3426SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3427 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3428
3429 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3430}
3431
3432SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3433 unsigned OpNo) {
3434 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3435 SDValue Op1 = N->getOperand(1);
3436 EVT RVT = Op1.getValueType();
3437 SDLoc dl(N);
3438
3439 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3440
3441 Op1 = GetSoftPromotedHalf(Op1);
3442 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3443
3444 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3445 Op1);
3446}
3447
3448SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3449 EVT RVT = N->getValueType(0);
3450 bool IsStrict = N->isStrictFPOpcode();
3451 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3452 EVT SVT = Op.getValueType();
3453 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3454
3455 if (IsStrict) {
3456 unsigned Opcode;
3457 if (SVT == MVT::f16)
3458 Opcode = ISD::STRICT_FP16_TO_FP;
3459 else if (SVT == MVT::bf16)
3460 Opcode = ISD::STRICT_BF16_TO_FP;
3461 else
3462 llvm_unreachable("unknown half type");
3463 SDValue Res =
3464 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3465 {N->getOperand(0), Op});
3466 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3467 ReplaceValueWith(SDValue(N, 0), Res);
3468 return SDValue();
3469 }
3470
3471 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3472}
3473
3474SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3475 EVT RVT = N->getValueType(0);
3476 SDValue Op = N->getOperand(0);
3477 EVT SVT = Op.getValueType();
3478 SDLoc dl(N);
3479
3480 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3481
3482 Op = GetSoftPromotedHalf(Op);
3483
3484 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3485
3486 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3487}
3488
3489SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3490 EVT RVT = N->getValueType(0);
3491 SDValue Op = N->getOperand(0);
3492 EVT SVT = Op.getValueType();
3493 SDLoc dl(N);
3494
3495 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3496
3497 Op = GetSoftPromotedHalf(Op);
3498
3499 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3500
3501 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3502 N->getOperand(1));
3503}
3504
3505SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3506 unsigned OpNo) {
3507 assert(OpNo == 0 && "Can only soften the comparison values");
3508 SDValue Op0 = N->getOperand(0);
3509 SDValue Op1 = N->getOperand(1);
3510 SDLoc dl(N);
3511
3512 EVT SVT = Op0.getValueType();
3513 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3514
3515 Op0 = GetSoftPromotedHalf(Op0);
3516 Op1 = GetSoftPromotedHalf(Op1);
3517
3518 // Promote to the larger FP type.
3519 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3520 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3521 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3522
3523 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3524 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3525}
3526
3527SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3528 SDValue Op0 = N->getOperand(0);
3529 SDValue Op1 = N->getOperand(1);
3530 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3531 SDLoc dl(N);
3532
3533 EVT SVT = Op0.getValueType();
3534 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3535
3536 Op0 = GetSoftPromotedHalf(Op0);
3537 Op1 = GetSoftPromotedHalf(Op1);
3538
3539 // Promote to the larger FP type.
3540 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3541 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3542 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3543
3544 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3545}
3546
3547SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3548 assert(OpNo == 1 && "Can only soften the stored value!");
3549 StoreSDNode *ST = cast<StoreSDNode>(N);
3550 SDValue Val = ST->getValue();
3551 SDLoc dl(N);
3552
3553 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3554 SDValue Promoted = GetSoftPromotedHalf(Val);
3555 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3556 ST->getMemOperand());
3557}
3558
3559SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3560 unsigned OpNo) {
3561 assert(OpNo == 1 && "Can only soften the stored value!");
3562 AtomicSDNode *ST = cast<AtomicSDNode>(N);
3563 SDValue Val = ST->getVal();
3564 SDLoc dl(N);
3565
3566 SDValue Promoted = GetSoftPromotedHalf(Val);
3567 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
3568 ST->getChain(), Promoted, ST->getBasePtr(),
3569 ST->getMemOperand());
3570}
3571
3572SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3573 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3574 SmallVector<SDValue> NewOps(N->ops());
3575 SDValue Op = N->getOperand(OpNo);
3576 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3577 SDValue NewNode =
3578 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3579
3580 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3581 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3582
3583 return SDValue(); // Signal that we replaced the node ourselves.
3584}
3585
3586SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3587 unsigned OpNo) {
3588 assert(OpNo >= 7);
3589 SmallVector<SDValue> NewOps(N->ops());
3590 SDValue Op = N->getOperand(OpNo);
3591 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3592 SDValue NewNode =
3593 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3594
3595 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3596 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3597
3598 return SDValue(); // Signal that we replaced the node ourselves.
3599}
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: ARMSLSHardening.cpp:73
Expansion
DXIL Intrinsic Expansion
Definition: DXILIntrinsicExpansion.cpp:429
Idx
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Definition: DeadArgumentElimination.cpp:352
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Size
uint64_t Size
Definition: ELFObjHandler.cpp:81
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:53
findFPToIntLibcall
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
Definition: LegalizeFloatTypes.cpp:1119
GetFPLibCall
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
Definition: LegalizeFloatTypes.cpp:32
GetPromotionOpcode
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2349
GetPromotionOpcodeStrict
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2363
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::APFloat::bitcastToAPInt
APInt bitcastToAPInt() const
Definition: APFloat.h:1266
llvm::APFloat::getZero
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition: APFloat.h:994
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:78
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:214
llvm::APInt::clearBit
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1387
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:209
llvm::APInt::getRawData
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:549
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
llvm::AtomicSDNode
This is an SDNode representing atomic operations.
Definition: SelectionDAGNodes.h:1496
llvm::AtomicSDNode::getVal
const SDValue & getVal() const
Definition: SelectionDAGNodes.h:1518
llvm::ConstantFPSDNode::getValueAPF
const APFloat & getValueAPF() const
Definition: SelectionDAGNodes.h:1718
llvm::DAGTypeLegalizer::NewNode
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
llvm::DWARFExpression::Operation
This class represents an Operation in the Expression.
Definition: DWARFExpression.h:32
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:206
llvm::LLVMContext::emitError
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Definition: LLVMContext.cpp:286
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2430
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:54
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:144
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:146
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1406
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1151
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:477
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:1028
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:1031
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1186
llvm::SelectionDAG::getExtLoad
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8973
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:10390
llvm::SelectionDAG::getShiftAmountConstant
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
Definition: SelectionDAG.cpp:1761
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1230
llvm::SelectionDAG::getConstantFP
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1817
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8956
llvm::SelectionDAG::getAtomic
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Definition: SelectionDAG.cpp:8677
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1115
llvm::SelectionDAG::getBitcast
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
Definition: SelectionDAG.cpp:2377
llvm::SelectionDAG::getSelect
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:1259
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:487
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1624
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:10007
llvm::SelectionDAG::getTruncStore
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:9057
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:9006
llvm::SelectionDAG::getSelectCC
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:1269
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1756
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:10013
llvm::SelectionDAG::getLibInfo
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:494
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:482
llvm::SelectionDAG::getCondCode
SDValue getCondCode(ISD::CondCode Cond)
Definition: SelectionDAG.cpp:2023
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:500
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2501
llvm::SelectionDAG::UpdateNodeOperands
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Definition: SelectionDAG.cpp:10480
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:570
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2458
llvm::TargetLibraryInfo::getIntSize
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Definition: TargetLibraryInfo.h:564
llvm::TargetLoweringBase::getShiftAmountTy
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
Definition: TargetLoweringBase.cpp:874
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1140
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:3435
llvm::TargetLowering::softenSetCCOperands
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
Definition: TargetLowering.cpp:291
llvm::TargetLowering::makeLibCall
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Definition: TargetLowering.cpp:146
llvm::TargetLowering::expandVecReduceSeq
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
Definition: TargetLowering.cpp:11196
llvm::TargetLowering::expandFP_TO_INT_SAT
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
Definition: TargetLowering.cpp:11247
llvm::TargetLowering::expandVecReduce
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
Definition: TargetLowering.cpp:11156
llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
Definition: TargetLowering.cpp:8421
uint64_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::ISD::NodeType
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:779
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:490
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1402
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::ISD::ATOMIC_STORE
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1304
llvm::ISD::STRICT_FCEIL
@ STRICT_FCEIL
Definition: ISDOpcodes.h:440
llvm::ISD::FMAD
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:501
llvm::ISD::STRICT_FTANH
@ STRICT_FTANH
Definition: ISDOpcodes.h:430
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1094
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:813
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:497
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:840
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1420
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
llvm::ISD::VECREDUCE_FMAXIMUM
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1424
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:491
llvm::ISD::FP16_TO_FP
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:963
llvm::ISD::STRICT_FLOG2
@ STRICT_FLOG2
Definition: ISDOpcodes.h:435
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:953
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:236
llvm::ISD::FLDEXP
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:996
llvm::ISD::STRICT_FSQRT
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:418
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:804
llvm::ISD::STRICT_FASIN
@ STRICT_FASIN
Definition: ISDOpcodes.h:425
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:464
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1417
llvm::ISD::STRICT_FATAN
@ STRICT_FATAN
Definition: ISDOpcodes.h:427
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1421
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:445
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:980